cleanup: move pkg/ to internal/

The internal/ directory in Go has a special meaning, and indicates that
those packages are not meant for external consumption. Ceph-CSI does
provide public APIs for other projects to consume. There is no plan to
keep the API of the internally used packages stable.

Closes: #903
Signed-off-by: Niels de Vos <ndevos@redhat.com>

internal/cephfs/cephfs_util.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ceph/ceph-csi/internal/util"
+)
+
+// MDSMap is a representation of the mds map sub-structure returned by 'ceph fs get'
+type MDSMap struct {
+	FilesystemName string `json:"fs_name"`
+}
+
+// CephFilesystemDetails is a representation of the main json structure returned by 'ceph fs get'
+type CephFilesystemDetails struct {
+	ID     int64  `json:"id"`
+	MDSMap MDSMap `json:"mdsmap"`
+}
+
+func getFscID(ctx context.Context, monitors string, cr *util.Credentials, fsName string) (int64, error) {
+	// ceph fs get myfs --format=json
+	// {"mdsmap":{...},"id":2}
+	var fsDetails CephFilesystemDetails
+	err := execCommandJSON(ctx, &fsDetails,
+		"ceph",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", util.CephConfigPath,
+		"fs", "get", fsName, "--format=json",
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	return fsDetails.ID, nil
+}
+
+// CephFilesystem is a representation of the json structure returned by 'ceph fs ls'
+type CephFilesystem struct {
+	Name           string   `json:"name"`
+	MetadataPool   string   `json:"metadata_pool"`
+	MetadataPoolID int      `json:"metadata_pool_id"`
+	DataPools      []string `json:"data_pools"`
+	DataPoolIDs    []int    `json:"data_pool_ids"`
+}
+
+func getMetadataPool(ctx context.Context, monitors string, cr *util.Credentials, fsName string) (string, error) {
+	// ./tbox ceph fs ls --format=json
+	// [{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,...},...]
+	var filesystems []CephFilesystem
+	err := execCommandJSON(ctx, &filesystems,
+		"ceph",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", util.CephConfigPath,
+		"fs", "ls", "--format=json",
+	)
+	if err != nil {
+		return "", err
+	}
+
+	for _, fs := range filesystems {
+		if fs.Name == fsName {
+			return fs.MetadataPool, nil
+		}
+	}
+
+	return "", util.ErrPoolNotFound{Pool: fsName, Err: fmt.Errorf("fsName (%s) not found in Ceph cluster", fsName)}
+}
+
+// CephFilesystemDump is a representation of the main json structure returned by 'ceph fs dump'
+type CephFilesystemDump struct {
+	Filesystems []CephFilesystemDetails `json:"filesystems"`
+}
+
+func getFsName(ctx context.Context, monitors string, cr *util.Credentials, fscID int64) (string, error) {
+	// ./tbox ceph fs dump --format=json
+	// JSON: {...,"filesystems":[{"mdsmap":{},"id":<n>},...],...}
+	var fsDump CephFilesystemDump
+	err := execCommandJSON(ctx, &fsDump,
+		"ceph",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", util.CephConfigPath,
+		"fs", "dump", "--format=json",
+	)
+	if err != nil {
+		return "", err
+	}
+
+	for _, fs := range fsDump.Filesystems {
+		if fs.ID == fscID {
+			return fs.MDSMap.FilesystemName, nil
+		}
+	}
+
+	return "", util.ErrPoolNotFound{Pool: string(fscID), Err: fmt.Errorf("fscID (%d) not found in Ceph cluster", fscID)}
+}

internal/cephfs/cephuser.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+
+	"github.com/ceph/ceph-csi/internal/util"
+)
+
+const (
+	cephUserPrefix         = "user-"
+	cephEntityClientPrefix = "client."
+)
+
+func genUserIDs(adminCr *util.Credentials, volID volumeID) (adminID, userID string) {
+	return cephEntityClientPrefix + adminCr.ID, cephEntityClientPrefix + getCephUserName(volID)
+}
+
+func getCephUserName(volID volumeID) string {
+	return cephUserPrefix + string(volID)
+}
+
+func deleteCephUserDeprecated(ctx context.Context, volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) error {
+	adminID, userID := genUserIDs(adminCr, volID)
+
+	// TODO: Need to return success if userID is not found
+	return execCommandErr(ctx, "ceph",
+		"-m", volOptions.Monitors,
+		"-n", adminID,
+		"--keyfile="+adminCr.KeyFile,
+		"-c", util.CephConfigPath,
+		"auth", "rm", userID,
+	)
+}

internal/cephfs/controllerserver.go

@@ -0,0 +1,373 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+// ControllerServer struct of CEPH CSI driver with supported methods of CSI
+// controller server spec.
+type ControllerServer struct {
+	*csicommon.DefaultControllerServer
+	MetadataStore util.CachePersister
+	// A map storing all volumes with ongoing operations so that additional operations
+	// for that same volume (as defined by VolumeID/volume name) return an Aborted error
+	VolumeLocks *util.VolumeLocks
+}
+
+type controllerCacheEntry struct {
+	VolOptions volumeOptions
+	VolumeID   volumeID
+}
+
+// createBackingVolume creates the backing subvolume and on any error cleans up any created entities
+func (cs *ControllerServer) createBackingVolume(ctx context.Context, volOptions *volumeOptions, vID *volumeIdentifier, secret map[string]string) error {
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	if err = createVolume(ctx, volOptions, cr, volumeID(vID.FsSubvolName), volOptions.Size); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create volume %s: %v"), volOptions.RequestName, err)
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	return nil
+}
+
+// CreateVolume creates a reservation and the volume in backend, if it is not already present
+func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
+	if err := cs.validateCreateVolumeRequest(req); err != nil {
+		klog.Errorf(util.Log(ctx, "CreateVolumeRequest validation failed: %v"), err)
+		return nil, err
+	}
+
+	// Configuration
+	secret := req.GetSecrets()
+	requestName := req.GetName()
+
+	// Existence and conflict checks
+	if acquired := cs.VolumeLocks.TryAcquire(requestName); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), requestName)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, requestName)
+	}
+	defer cs.VolumeLocks.Release(requestName)
+
+	volOptions, err := newVolumeOptions(ctx, requestName, req, secret)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "validation and extraction of volume options failed: %v"), err)
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	if req.GetCapacityRange() != nil {
+		volOptions.Size = util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes())
+	}
+	// TODO need to add check for 0 volume size
+
+	vID, err := checkVolExists(ctx, volOptions, secret)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	// TODO return error message if requested vol size greater than found volume return error
+
+	if vID != nil {
+		volumeContext := req.GetParameters()
+		volumeContext["subvolumeName"] = vID.FsSubvolName
+		volume := &csi.Volume{
+			VolumeId:      vID.VolumeID,
+			CapacityBytes: volOptions.Size,
+			VolumeContext: volumeContext,
+		}
+		if volOptions.Topology != nil {
+			volume.AccessibleTopology =
+				[]*csi.Topology{
+					{
+						Segments: volOptions.Topology,
+					},
+				}
+		}
+		return &csi.CreateVolumeResponse{Volume: volume}, nil
+	}
+
+	// Reservation
+	vID, err = reserveVol(ctx, volOptions, secret)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			errDefer := undoVolReservation(ctx, volOptions, *vID, secret)
+			if errDefer != nil {
+				klog.Warningf(util.Log(ctx, "failed undoing reservation of volume: %s (%s)"),
+					requestName, errDefer)
+			}
+		}
+	}()
+
+	// Create a volume
+	err = cs.createBackingVolume(ctx, volOptions, vID, secret)
+	if err != nil {
+		return nil, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully created backing volume named %s for request name %s"),
+		vID.FsSubvolName, requestName)
+
+	volumeContext := req.GetParameters()
+	volumeContext["subvolumeName"] = vID.FsSubvolName
+	volume := &csi.Volume{
+		VolumeId:      vID.VolumeID,
+		CapacityBytes: volOptions.Size,
+		VolumeContext: volumeContext,
+	}
+	if volOptions.Topology != nil {
+		volume.AccessibleTopology =
+			[]*csi.Topology{
+				{
+					Segments: volOptions.Topology,
+				},
+			}
+	}
+	return &csi.CreateVolumeResponse{Volume: volume}, nil
+}
+
+// deleteVolumeDeprecated is used to delete volumes created using version 1.0.0 of the plugin,
+// that have state information stored in files or kubernetes config maps
+func (cs *ControllerServer) deleteVolumeDeprecated(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
+	var (
+		volID   = volumeID(req.GetVolumeId())
+		secrets = req.GetSecrets()
+	)
+
+	ce := &controllerCacheEntry{}
+	if err := cs.MetadataStore.Get(string(volID), ce); err != nil {
+		if err, ok := err.(*util.CacheEntryNotFound); ok {
+			klog.Warningf(util.Log(ctx, "cephfs: metadata for volume %s not found, assuming the volume to be already deleted (%v)"), volID, err)
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if !ce.VolOptions.ProvisionVolume {
+		// DeleteVolume() is forbidden for statically provisioned volumes!
+
+		klog.Warningf(util.Log(ctx, "volume %s is provisioned statically, aborting delete"), volID)
+		return &csi.DeleteVolumeResponse{}, nil
+	}
+
+	// mons may have changed since create volume,
+	// retrieve the latest mons and override old mons
+	if mon, secretsErr := util.GetMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 {
+		klog.V(3).Infof(util.Log(ctx, "overriding monitors [%q] with [%q] for volume %s"), ce.VolOptions.Monitors, mon, volID)
+		ce.VolOptions.Monitors = mon
+	}
+
+	// Deleting a volume requires admin credentials
+
+	cr, err := util.NewAdminCredentials(secrets)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to retrieve admin credentials: %v"), err)
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	if acquired := cs.VolumeLocks.TryAcquire(string(volID)); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, string(volID))
+	}
+	defer cs.VolumeLocks.Release(string(volID))
+
+	if err = purgeVolumeDeprecated(ctx, volID, cr, &ce.VolOptions); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to delete volume %s: %v"), volID, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if err = deleteCephUserDeprecated(ctx, &ce.VolOptions, cr, volID); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to delete ceph user for volume %s: %v"), volID, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if err = cs.MetadataStore.Delete(string(volID)); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully deleted volume %s"), volID)
+
+	return &csi.DeleteVolumeResponse{}, nil
+}
+
+// DeleteVolume deletes the volume in backend and its reservation
+func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
+	if err := cs.validateDeleteVolumeRequest(); err != nil {
+		klog.Errorf(util.Log(ctx, "DeleteVolumeRequest validation failed: %v"), err)
+		return nil, err
+	}
+
+	volID := volumeID(req.GetVolumeId())
+	secrets := req.GetSecrets()
+
+	// lock out parallel delete operations
+	if acquired := cs.VolumeLocks.TryAcquire(string(volID)); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, string(volID))
+	}
+	defer cs.VolumeLocks.Release(string(volID))
+
+	// Find the volume using the provided VolumeID
+	volOptions, vID, err := newVolumeOptionsFromVolID(ctx, string(volID), nil, secrets)
+	if err != nil {
+		// if error is ErrPoolNotFound, the pool is already deleted we dont
+		// need to worry about deleting subvolume or omap data, return success
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Warningf(util.Log(ctx, "failed to get backend volume for %s: %v"), string(volID), err)
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+		// if error is ErrKeyNotFound, then a previous attempt at deletion was complete
+		// or partially complete (subvolume and imageOMap are garbage collected already), hence
+		// return success as deletion is complete
+		if _, ok := err.(util.ErrKeyNotFound); ok {
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		// ErrInvalidVolID may mean this is an 1.0.0 version volume
+		if _, ok := err.(ErrInvalidVolID); ok && cs.MetadataStore != nil {
+			return cs.deleteVolumeDeprecated(ctx, req)
+		}
+
+		// All errors other than ErrVolumeNotFound should return an error back to the caller
+		if _, ok := err.(ErrVolumeNotFound); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// If error is ErrImageNotFound then we failed to find the subvolume, but found the imageOMap
+		// to lead us to the image, hence the imageOMap needs to be garbage collected, by calling
+		// unreserve for the same
+		if acquired := cs.VolumeLocks.TryAcquire(volOptions.RequestName); !acquired {
+			return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volOptions.RequestName)
+		}
+		defer cs.VolumeLocks.Release(volOptions.RequestName)
+
+		if err = undoVolReservation(ctx, volOptions, *vID, secrets); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		return &csi.DeleteVolumeResponse{}, nil
+	}
+
+	// lock out parallel delete and create requests against the same volume name as we
+	// cleanup the subvolume and associated omaps for the same
+	if acquired := cs.VolumeLocks.TryAcquire(volOptions.RequestName); !acquired {
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volOptions.RequestName)
+	}
+	defer cs.VolumeLocks.Release(string(volID))
+
+	// Deleting a volume requires admin credentials
+	cr, err := util.NewAdminCredentials(secrets)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to retrieve admin credentials: %v"), err)
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	if err = purgeVolume(ctx, volumeID(vID.FsSubvolName), cr, volOptions); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to delete volume %s: %v"), volID, err)
+		// All errors other than ErrVolumeNotFound should return an error back to the caller
+		if _, ok := err.(ErrVolumeNotFound); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	if err := undoVolReservation(ctx, volOptions, *vID, secrets); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully deleted volume %s"), volID)
+
+	return &csi.DeleteVolumeResponse{}, nil
+}
+
+// ValidateVolumeCapabilities checks whether the volume capabilities requested
+// are supported.
+func (cs *ControllerServer) ValidateVolumeCapabilities(
+	ctx context.Context,
+	req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
+	// Cephfs doesn't support Block volume
+	for _, cap := range req.VolumeCapabilities {
+		if cap.GetBlock() != nil {
+			return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
+		}
+	}
+	return &csi.ValidateVolumeCapabilitiesResponse{
+		Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
+			VolumeCapabilities: req.VolumeCapabilities,
+		},
+	}, nil
+}
+
+// ControllerExpandVolume expands CephFS Volumes on demand based on resizer request
+func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	if err := cs.validateExpandVolumeRequest(req); err != nil {
+		klog.Errorf(util.Log(ctx, "ControllerExpandVolumeRequest validation failed: %v"), err)
+		return nil, err
+	}
+
+	volID := req.GetVolumeId()
+	secret := req.GetSecrets()
+
+	// lock out parallel delete operations
+	if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer cs.VolumeLocks.Release(volID)
+
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	volOptions, volIdentifier, err := newVolumeOptionsFromVolID(ctx, volID, nil, secret)
+
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "validation and extraction of volume options failed: %v"), err)
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	RoundOffSize := util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes())
+
+	if err = createVolume(ctx, volOptions, cr, volumeID(volIdentifier.FsSubvolName), RoundOffSize); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to expand volume %s: %v"), volumeID(volIdentifier.FsSubvolName), err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	return &csi.ControllerExpandVolumeResponse{
+		CapacityBytes:         RoundOffSize,
+		NodeExpansionRequired: false,
+	}, nil
+}

internal/cephfs/driver.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"k8s.io/klog"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+const (
+	// volIDVersion is the version number of volume ID encoding scheme
+	volIDVersion uint16 = 1
+
+	// csiConfigFile is the location of the CSI config file
+	csiConfigFile = "/etc/ceph-csi-config/config.json"
+
+	// RADOS namespace to store CSI specific objects and keys
+	radosNamespace = "csi"
+)
+
+// PluginFolder defines the location of ceph plugin
+var PluginFolder = ""
+
+// Driver contains the default identity,node and controller struct
+type Driver struct {
+	cd *csicommon.CSIDriver
+
+	is *IdentityServer
+	ns *NodeServer
+	cs *ControllerServer
+}
+
+var (
+	// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
+	// ceph clusters across CSI instances, to differentiate omap names per CSI instance
+	CSIInstanceID = "default"
+
+	// volJournal is used to maintain RADOS based journals for CO generated
+	// VolumeName to backing CephFS subvolumes
+	volJournal *util.CSIJournal
+)
+
+// NewDriver returns new ceph driver
+func NewDriver() *Driver {
+	return &Driver{}
+}
+
+// NewIdentityServer initialize a identity server for ceph CSI driver
+func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer {
+	return &IdentityServer{
+		DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
+	}
+}
+
+// NewControllerServer initialize a controller server for ceph CSI driver
+func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer {
+	return &ControllerServer{
+		DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
+		MetadataStore:           cachePersister,
+		VolumeLocks:             util.NewVolumeLocks(),
+	}
+}
+
+// NewNodeServer initialize a node server for ceph CSI driver.
+func NewNodeServer(d *csicommon.CSIDriver, t string, topology map[string]string) *NodeServer {
+	return &NodeServer{
+		DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t, topology),
+		VolumeLocks:       util.NewVolumeLocks(),
+	}
+}
+
+// Run start a non-blocking grpc controller,node and identityserver for
+// ceph CSI driver which can serve multiple parallel requests
+func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
+	var err error
+	var topology map[string]string
+
+	// Configuration
+	PluginFolder = conf.PluginPath
+
+	if err = loadAvailableMounters(conf); err != nil {
+		klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
+	}
+
+	if err = util.WriteCephConfig(); err != nil {
+		klog.Fatalf("failed to write ceph configuration file: %v", err)
+	}
+
+	// Use passed in instance ID, if provided for omap suffix naming
+	if conf.InstanceID != "" {
+		CSIInstanceID = conf.InstanceID
+	}
+	// Get an instance of the volume journal
+	volJournal = util.NewCSIVolumeJournal()
+
+	// Update keys with CSI instance suffix
+	volJournal.SetCSIDirectorySuffix(CSIInstanceID)
+
+	// Update namespace for storing keys into a specific namespace on RADOS, in the CephFS
+	// metadata pool
+	volJournal.SetNamespace(radosNamespace)
+	// Initialize default library driver
+
+	fs.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID)
+	if fs.cd == nil {
+		klog.Fatalln("failed to initialize CSI driver")
+	}
+
+	if conf.IsControllerServer || !conf.IsNodeServer {
+		fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
+			csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
+			csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
+		})
+
+		fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
+			csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
+		})
+	}
+	// Create gRPC servers
+
+	fs.is = NewIdentityServer(fs.cd)
+
+	if conf.IsNodeServer {
+		topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName)
+		if err != nil {
+			klog.Fatalln(err)
+		}
+		fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology)
+	}
+
+	if conf.IsControllerServer {
+		fs.cs = NewControllerServer(fs.cd, cachePersister)
+	}
+	if !conf.IsControllerServer && !conf.IsNodeServer {
+		topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName)
+		if err != nil {
+			klog.Fatalln(err)
+		}
+		fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology)
+		fs.cs = NewControllerServer(fs.cd, cachePersister)
+	}
+
+	server := csicommon.NewNonBlockingGRPCServer()
+	server.Start(conf.Endpoint, conf.HistogramOption, fs.is, fs.cs, fs.ns, conf.EnableGRPCMetrics)
+	if conf.EnableGRPCMetrics {
+		klog.Warning("EnableGRPCMetrics is deprecated")
+		go util.StartMetricsServer(conf)
+	}
+	server.Wait()
+}

internal/cephfs/errors.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+// ErrInvalidVolID is returned when a CSI passed VolumeID is not conformant to any known volume ID
+// formats
+type ErrInvalidVolID struct {
+	err error
+}
+
+func (e ErrInvalidVolID) Error() string {
+	return e.err.Error()
+}
+
+// ErrNonStaticVolume is returned when a volume is detected as not being
+// statically provisioned
+type ErrNonStaticVolume struct {
+	err error
+}
+
+func (e ErrNonStaticVolume) Error() string {
+	return e.err.Error()
+}
+
+// ErrVolumeNotFound is returned when a subvolume is not found in CephFS
+type ErrVolumeNotFound struct {
+	err error
+}
+
+func (e ErrVolumeNotFound) Error() string {
+	return e.err.Error()
+}

internal/cephfs/fsjournal.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"k8s.io/klog"
+)
+
+// volumeIdentifier structure contains an association between the CSI VolumeID to its subvolume
+// name on the backing CephFS instance
+type volumeIdentifier struct {
+	FsSubvolName string
+	VolumeID     string
+}
+
+/*
+checkVolExists checks to determine if passed in RequestName in volOptions exists on the backend.
+
+**NOTE:** These functions manipulate the rados omaps that hold information regarding
+volume names as requested by the CSI drivers. Hence, these need to be invoked only when the
+respective CSI driver generated volume name based locks are held, as otherwise racy
+access to these omaps may end up leaving them in an inconsistent state.
+
+These functions also cleanup omap reservations that are stale. I.e when omap entries exist and
+backing subvolumes are missing, or one of the omaps exist and the next is missing. This is
+because, the order of omap creation and deletion are inverse of each other, and protected by the
+request name lock, and hence any stale omaps are leftovers from incomplete transactions and are
+hence safe to garbage collect.
+*/
+func checkVolExists(ctx context.Context, volOptions *volumeOptions, secret map[string]string) (*volumeIdentifier, error) {
+	var vid volumeIdentifier
+
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return nil, err
+	}
+	defer cr.DeleteCredentials()
+
+	imageData, err := volJournal.CheckReservation(ctx, volOptions.Monitors, cr,
+		volOptions.MetadataPool, volOptions.RequestName, volOptions.NamePrefix, "", "")
+	if err != nil {
+		return nil, err
+	}
+	if imageData == nil {
+		return nil, nil
+	}
+	imageUUID := imageData.ImageUUID
+	vid.FsSubvolName = imageData.ImageAttributes.ImageName
+
+	_, err = getVolumeRootPathCeph(ctx, volOptions, cr, volumeID(vid.FsSubvolName))
+	if err != nil {
+		if _, ok := err.(ErrVolumeNotFound); ok {
+			err = volJournal.UndoReservation(ctx, volOptions.Monitors, cr, volOptions.MetadataPool,
+				volOptions.MetadataPool, vid.FsSubvolName, volOptions.RequestName)
+			return nil, err
+		}
+		return nil, err
+	}
+
+	// check if topology constraints match what is found
+	// TODO: we need an API to fetch subvolume attributes (size/datapool and others), based
+	// on which we can evaluate which topology this belongs to.
+	// TODO: CephFS topology support is postponed till we get the same
+	// TODO: size checks
+
+	// found a volume already available, process and return it!
+	vid.VolumeID, err = util.GenerateVolID(ctx, volOptions.Monitors, cr, volOptions.FscID,
+		"", volOptions.ClusterID, imageUUID, volIDVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "Found existing volume (%s) with subvolume name (%s) for request (%s)"),
+		vid.VolumeID, vid.FsSubvolName, volOptions.RequestName)
+
+	return &vid, nil
+}
+
+// undoVolReservation is a helper routine to undo a name reservation for a CSI VolumeName
+func undoVolReservation(ctx context.Context, volOptions *volumeOptions, vid volumeIdentifier, secret map[string]string) error {
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return err
+	}
+	defer cr.DeleteCredentials()
+
+	err = volJournal.UndoReservation(ctx, volOptions.Monitors, cr, volOptions.MetadataPool,
+		volOptions.MetadataPool, vid.FsSubvolName, volOptions.RequestName)
+
+	return err
+}
+
+func updateTopologyConstraints(volOpts *volumeOptions) error {
+	// update request based on topology constrained parameters (if present)
+	poolName, _, topology, err := util.FindPoolAndTopology(volOpts.TopologyPools, volOpts.TopologyRequirement)
+	if err != nil {
+		return err
+	}
+	if poolName != "" {
+		volOpts.Pool = poolName
+		volOpts.Topology = topology
+	}
+
+	return nil
+}
+
+// reserveVol is a helper routine to request a UUID reservation for the CSI VolumeName and,
+// to generate the volume identifier for the reserved UUID
+func reserveVol(ctx context.Context, volOptions *volumeOptions, secret map[string]string) (*volumeIdentifier, error) {
+	var (
+		vid       volumeIdentifier
+		imageUUID string
+		err       error
+	)
+
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return nil, err
+	}
+	defer cr.DeleteCredentials()
+
+	err = updateTopologyConstraints(volOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	imageUUID, vid.FsSubvolName, err = volJournal.ReserveName(ctx, volOptions.Monitors, cr, volOptions.MetadataPool, util.InvalidPoolID,
+		volOptions.MetadataPool, util.InvalidPoolID, volOptions.RequestName, volOptions.NamePrefix, "", "")
+	if err != nil {
+		return nil, err
+	}
+
+	// generate the volume ID to return to the CO system
+	vid.VolumeID, err = util.GenerateVolID(ctx, volOptions.Monitors, cr, volOptions.FscID,
+		"", volOptions.ClusterID, imageUUID, volIDVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "Generated Volume ID (%s) and subvolume name (%s) for request name (%s)"),
+		vid.VolumeID, vid.FsSubvolName, volOptions.RequestName)
+
+	return &vid, nil
+}

internal/cephfs/identityserver.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+// IdentityServer struct of ceph CSI driver with supported methods of CSI
+// identity server spec.
+type IdentityServer struct {
+	*csicommon.DefaultIdentityServer
+}
+
+// GetPluginCapabilities returns available capabilities of the ceph driver
+func (is *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
+	return &csi.GetPluginCapabilitiesResponse{
+		Capabilities: []*csi.PluginCapability{
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
+					},
+				},
+			},
+			{
+				Type: &csi.PluginCapability_VolumeExpansion_{
+					VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
+						Type: csi.PluginCapability_VolumeExpansion_ONLINE,
+					},
+				},
+			},
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS,
+					},
+				},
+			},
+		},
+	}, nil
+}

internal/cephfs/nodeserver.go

@@ -0,0 +1,297 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+// NodeServer struct of ceph CSI driver with supported methods of CSI
+// node server spec.
+type NodeServer struct {
+	*csicommon.DefaultNodeServer
+	// A map storing all volumes with ongoing operations so that additional operations
+	// for that same volume (as defined by VolumeID) return an Aborted error
+	VolumeLocks *util.VolumeLocks
+}
+
+func getCredentialsForVolume(volOptions *volumeOptions, req *csi.NodeStageVolumeRequest) (*util.Credentials, error) {
+	var (
+		err     error
+		cr      *util.Credentials
+		secrets = req.GetSecrets()
+	)
+
+	if volOptions.ProvisionVolume {
+		// The volume is provisioned dynamically, use passed in admin credentials
+
+		cr, err = util.NewAdminCredentials(secrets)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
+		}
+	} else {
+		// The volume is pre-made, credentials are in node stage secrets
+
+		cr, err = util.NewUserCredentials(req.GetSecrets())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
+		}
+	}
+
+	return cr, nil
+}
+
+// NodeStageVolume mounts the volume to a staging path on the node.
+func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+	var (
+		volOptions *volumeOptions
+	)
+	if err := util.ValidateNodeStageVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	// Configuration
+
+	stagingTargetPath := req.GetStagingTargetPath()
+	volID := volumeID(req.GetVolumeId())
+
+	if acquired := ns.VolumeLocks.TryAcquire(req.GetVolumeId()); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetVolumeId())
+	}
+	defer ns.VolumeLocks.Release(req.GetVolumeId())
+
+	volOptions, _, err := newVolumeOptionsFromVolID(ctx, string(volID), req.GetVolumeContext(), req.GetSecrets())
+	if err != nil {
+		if _, ok := err.(ErrInvalidVolID); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// check for pre-provisioned volumes (plugin versions > 1.0.0)
+		volOptions, _, err = newVolumeOptionsFromStaticVolume(string(volID), req.GetVolumeContext())
+		if err != nil {
+			if _, ok := err.(ErrNonStaticVolume); !ok {
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+
+			// check for volumes from plugin versions <= 1.0.0
+			volOptions, _, err = newVolumeOptionsFromVersion1Context(string(volID), req.GetVolumeContext(),
+				req.GetSecrets())
+			if err != nil {
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+		}
+	}
+
+	// Check if the volume is already mounted
+
+	isMnt, err := util.IsMountPoint(stagingTargetPath)
+
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "stat failed: %v"), err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if isMnt {
+		klog.V(4).Infof(util.Log(ctx, "cephfs: volume %s is already mounted to %s, skipping"), volID, stagingTargetPath)
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
+	// It's not, mount now
+	if err = ns.mount(ctx, volOptions, req); err != nil {
+		return nil, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully mounted volume %s to %s"), volID, stagingTargetPath)
+
+	return &csi.NodeStageVolumeResponse{}, nil
+}
+
+func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *csi.NodeStageVolumeRequest) error {
+	stagingTargetPath := req.GetStagingTargetPath()
+	volID := volumeID(req.GetVolumeId())
+
+	cr, err := getCredentialsForVolume(volOptions, req)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get ceph credentials for volume %s: %v"), volID, err)
+		return status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	m, err := newMounter(volOptions)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create mounter for volume %s: %v"), volID, err)
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: mounting volume %s with %s"), volID, m.name())
+
+	if err = m.mount(ctx, stagingTargetPath, cr, volOptions); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to mount volume %s: %v"), volID, err)
+		return status.Error(codes.Internal, err.Error())
+	}
+	return nil
+}
+
+// NodePublishVolume mounts the volume mounted to the staging path to the target
+// path
+func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
+	mountOptions := []string{"bind", "_netdev"}
+	if err := util.ValidateNodePublishVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	targetPath := req.GetTargetPath()
+	volID := req.GetVolumeId()
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	if err := util.CreateMountPoint(targetPath); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create mount point at %s: %v"), targetPath, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if req.GetReadonly() {
+		mountOptions = append(mountOptions, "ro")
+	}
+
+	mountOptions = csicommon.ConstructMountOptions(mountOptions, req.GetVolumeCapability())
+
+	// Check if the volume is already mounted
+
+	isMnt, err := util.IsMountPoint(targetPath)
+
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "stat failed: %v"), err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if isMnt {
+		klog.V(4).Infof(util.Log(ctx, "cephfs: volume %s is already bind-mounted to %s"), volID, targetPath)
+		return &csi.NodePublishVolumeResponse{}, nil
+	}
+
+	// It's not, mount now
+
+	if err = bindMount(ctx, req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly(), mountOptions); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to bind-mount volume %s: %v"), volID, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully bind-mounted volume %s to %s"), volID, targetPath)
+
+	// #nosec - allow anyone to write inside the target path
+	err = os.Chmod(targetPath, 0777)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to change targetpath permission for volume %s: %v"), volID, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	return &csi.NodePublishVolumeResponse{}, nil
+}
+
+// NodeUnpublishVolume unmounts the volume from the target path
+func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
+	var err error
+	if err = util.ValidateNodeUnpublishVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	volID := req.GetVolumeId()
+	targetPath := req.GetTargetPath()
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	// Unmount the bind-mount
+	if err = unmountVolume(ctx, targetPath); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	err = os.Remove(targetPath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully unbinded volume %s from %s"), req.GetVolumeId(), targetPath)
+
+	return &csi.NodeUnpublishVolumeResponse{}, nil
+}
+
+// NodeUnstageVolume unstages the volume from the staging path
+func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	var err error
+	if err = util.ValidateNodeUnstageVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	volID := req.GetVolumeId()
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	stagingTargetPath := req.GetStagingTargetPath()
+	// Unmount the volume
+	if err = unmountVolume(ctx, stagingTargetPath); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: successfully unmounted volume %s from %s"), req.GetVolumeId(), stagingTargetPath)
+
+	return &csi.NodeUnstageVolumeResponse{}, nil
+}
+
+// NodeGetCapabilities returns the supported capabilities of the node server
+func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
+	return &csi.NodeGetCapabilitiesResponse{
+		Capabilities: []*csi.NodeServiceCapability{
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
+					},
+				},
+			},
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS,
+					},
+				},
+			},
+		},
+	}, nil
+}

internal/cephfs/util.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+type volumeID string
+
+func execCommand(ctx context.Context, program string, args ...string) (stdout, stderr []byte, err error) {
+	var (
+		cmd           = exec.Command(program, args...) // nolint: gosec, #nosec
+		sanitizedArgs = util.StripSecretInArgs(args)
+		stdoutBuf     bytes.Buffer
+		stderrBuf     bytes.Buffer
+	)
+
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = &stderrBuf
+
+	klog.V(4).Infof(util.Log(ctx, "cephfs: EXEC %s %s"), program, sanitizedArgs)
+
+	if err := cmd.Run(); err != nil {
+		if cmd.Process == nil {
+			return nil, nil, fmt.Errorf("cannot get process pid while running %s %v: %v: %s",
+				program, sanitizedArgs, err, stderrBuf.Bytes())
+		}
+		return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
+			cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
+	}
+
+	return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
+}
+
+func execCommandErr(ctx context.Context, program string, args ...string) error {
+	_, _, err := execCommand(ctx, program, args...)
+	return err
+}
+
+//nolint: unparam
+func execCommandJSON(ctx context.Context, v interface{}, program string, args ...string) error {
+	stdout, _, err := execCommand(ctx, program, args...)
+	if err != nil {
+		return err
+	}
+
+	if err = json.Unmarshal(stdout, v); err != nil {
+		return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
+	}
+
+	return nil
+}
+
+func pathExists(p string) bool {
+	_, err := os.Stat(p)
+	return err == nil
+}
+
+// Controller service request validation
+func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+		return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
+	}
+
+	if req.GetName() == "" {
+		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
+	}
+
+	reqCaps := req.GetVolumeCapabilities()
+	if reqCaps == nil {
+		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
+	}
+
+	for _, cap := range reqCaps {
+		if cap.GetBlock() != nil {
+			return status.Error(codes.Unimplemented, "block volume not supported")
+		}
+	}
+
+	return nil
+}
+
+func (cs *ControllerServer) validateDeleteVolumeRequest() error {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+		return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
+	}
+
+	return nil
+}
+
+// Controller expand volume request validation
+func (cs *ControllerServer) validateExpandVolumeRequest(req *csi.ControllerExpandVolumeRequest) error {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil {
+		return fmt.Errorf("invalid ExpandVolumeRequest: %v", err)
+	}
+
+	if req.GetVolumeId() == "" {
+		return status.Error(codes.InvalidArgument, "Volume ID cannot be empty")
+	}
+
+	capRange := req.GetCapacityRange()
+	if capRange == nil {
+		return status.Error(codes.InvalidArgument, "CapacityRange cannot be empty")
+	}
+
+	return nil
+}

internal/cephfs/volume.go

@@ -0,0 +1,231 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"k8s.io/klog"
+)
+
+const (
+	csiSubvolumeGroup = "csi"
+)
+
+var (
+	// cephfsInit is used to create "csi" subvolume group for the first time the csi plugin loads.
+	// Subvolume group create gets called every time the plugin loads, though it doesn't result in error
+	// its unnecessary
+	cephfsInit = false
+)
+
+func getCephRootVolumePathLocalDeprecated(volID volumeID) string {
+	return path.Join(getCephRootPathLocalDeprecated(volID), "csi-volumes", string(volID))
+}
+
+func getVolumeRootPathCephDeprecated(volID volumeID) string {
+	return path.Join("/", "csi-volumes", string(volID))
+}
+
+func getCephRootPathLocalDeprecated(volID volumeID) string {
+	return fmt.Sprintf("%s/controller/volumes/root-%s", PluginFolder, string(volID))
+}
+
+func getVolumeNotFoundErrorString(volID volumeID) string {
+	return fmt.Sprintf("Error ENOENT: Subvolume '%s' not found", string(volID))
+}
+
+func getVolumeRootPathCeph(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID) (string, error) {
+	stdout, stderr, err := util.ExecCommand(
+		"ceph",
+		"fs",
+		"subvolume",
+		"getpath",
+		volOptions.FsName,
+		string(volID),
+		"--group_name",
+		csiSubvolumeGroup,
+		"-m", volOptions.Monitors,
+		"-c", util.CephConfigPath,
+		"-n", cephEntityClientPrefix+cr.ID,
+		"--keyfile="+cr.KeyFile)
+
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get the rootpath for the vol %s(%s)"), string(volID), err)
+
+		if strings.Contains(string(stderr), getVolumeNotFoundErrorString(volID)) {
+			return "", ErrVolumeNotFound{err}
+		}
+
+		return "", err
+	}
+	return strings.TrimSuffix(string(stdout), "\n"), nil
+}
+
+func createVolume(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID, bytesQuota int64) error {
+	//TODO: When we support multiple fs, need to hande subvolume group create for all fs's
+	if !cephfsInit {
+		err := execCommandErr(
+			ctx,
+			"ceph",
+			"fs",
+			"subvolumegroup",
+			"create",
+			volOptions.FsName,
+			csiSubvolumeGroup,
+			"-m", volOptions.Monitors,
+			"-c", util.CephConfigPath,
+			"-n", cephEntityClientPrefix+cr.ID,
+			"--keyfile="+cr.KeyFile)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to create subvolume group csi, for the vol %s(%s)"), string(volID), err)
+			return err
+		}
+		klog.V(4).Infof(util.Log(ctx, "cephfs: created subvolume group csi"))
+		cephfsInit = true
+	}
+
+	args := []string{
+		"fs",
+		"subvolume",
+		"create",
+		volOptions.FsName,
+		string(volID),
+		strconv.FormatInt(bytesQuota, 10),
+		"--group_name",
+		csiSubvolumeGroup,
+		"--mode", "777",
+		"-m", volOptions.Monitors,
+		"-c", util.CephConfigPath,
+		"-n", cephEntityClientPrefix + cr.ID,
+		"--keyfile=" + cr.KeyFile,
+	}
+
+	if volOptions.Pool != "" {
+		args = append(args, "--pool_layout", volOptions.Pool)
+	}
+
+	err := execCommandErr(
+		ctx,
+		"ceph",
+		args[:]...)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create subvolume %s(%s) in fs %s"), string(volID), err, volOptions.FsName)
+		return err
+	}
+
+	return nil
+}
+
+func mountCephRoot(ctx context.Context, volID volumeID, volOptions *volumeOptions, adminCr *util.Credentials) error {
+	cephRoot := getCephRootPathLocalDeprecated(volID)
+
+	// Root path is not set for dynamically provisioned volumes
+	// Access to cephfs's / is required
+	volOptions.RootPath = "/"
+
+	if err := util.CreateMountPoint(cephRoot); err != nil {
+		return err
+	}
+
+	m, err := newMounter(volOptions)
+	if err != nil {
+		return fmt.Errorf("failed to create mounter: %v", err)
+	}
+
+	if err = m.mount(ctx, cephRoot, adminCr, volOptions); err != nil {
+		return fmt.Errorf("error mounting ceph root: %v", err)
+	}
+
+	return nil
+}
+
+func unmountCephRoot(ctx context.Context, volID volumeID) {
+	cephRoot := getCephRootPathLocalDeprecated(volID)
+
+	if err := unmountVolume(ctx, cephRoot); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to unmount %s with error %s"), cephRoot, err)
+	} else {
+		if err := os.Remove(cephRoot); err != nil {
+			klog.Errorf(util.Log(ctx, "failed to remove %s with error %s"), cephRoot, err)
+		}
+	}
+}
+
+func purgeVolumeDeprecated(ctx context.Context, volID volumeID, adminCr *util.Credentials, volOptions *volumeOptions) error {
+	if err := mountCephRoot(ctx, volID, volOptions, adminCr); err != nil {
+		return err
+	}
+	defer unmountCephRoot(ctx, volID)
+
+	var (
+		volRoot         = getCephRootVolumePathLocalDeprecated(volID)
+		volRootDeleting = volRoot + "-deleting"
+	)
+
+	if pathExists(volRoot) {
+		if err := os.Rename(volRoot, volRootDeleting); err != nil {
+			return fmt.Errorf("couldn't mark volume %s for deletion: %v", volID, err)
+		}
+	} else {
+		if !pathExists(volRootDeleting) {
+			klog.V(4).Infof(util.Log(ctx, "cephfs: volume %s not found, assuming it to be already deleted"), volID)
+			return nil
+		}
+	}
+
+	if err := os.RemoveAll(volRootDeleting); err != nil {
+		return fmt.Errorf("failed to delete volume %s: %v", volID, err)
+	}
+
+	return nil
+}
+
+func purgeVolume(ctx context.Context, volID volumeID, cr *util.Credentials, volOptions *volumeOptions) error {
+	err := execCommandErr(
+		ctx,
+		"ceph",
+		"fs",
+		"subvolume",
+		"rm",
+		volOptions.FsName,
+		string(volID),
+		"--group_name",
+		csiSubvolumeGroup,
+		"-m", volOptions.Monitors,
+		"-c", util.CephConfigPath,
+		"-n", cephEntityClientPrefix+cr.ID,
+		"--keyfile="+cr.KeyFile)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to purge subvolume %s(%s) in fs %s"), string(volID), err, volOptions.FsName)
+
+		if strings.Contains(err.Error(), getVolumeNotFoundErrorString(volID)) {
+			return ErrVolumeNotFound{err}
+		}
+
+		return err
+	}
+
+	return nil
+}

internal/cephfs/volumemounter.go

@@ -0,0 +1,362 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/klog"
+)
+
+const (
+	volumeMounterFuse   = "fuse"
+	volumeMounterKernel = "kernel"
+	netDev              = "_netdev"
+)
+
+var (
+	availableMounters []string
+
+	// maps a mountpoint to PID of its FUSE daemon
+	fusePidMap    = make(map[string]int)
+	fusePidMapMtx sync.Mutex
+
+	fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`)
+)
+
+// Version checking the running kernel and comparing it to known versions that
+// have support for quota. Distributors of enterprise Linux have backported
+// quota support to previous versions. This function checks if the running
+// kernel is one of the versions that have the feature/fixes backported.
+//
+// `uname -r` (or Uname().Utsname.Release has a format like 1.2.3-rc.vendor
+// This can be slit up in the following components:
+// - version (1)
+// - patchlevel (2)
+// - sublevel (3) - optional, defaults to 0
+// - extraversion (rc) - optional, matching integers only
+// - distribution (.vendor) - optional, match against whole `uname -r` string
+//
+// For matching multiple versions, the kernelSupport type contains a backport
+// bool, which will cause matching
+// version+patchlevel+sublevel+(>=extraversion)+(~distribution)
+//
+// In case the backport bool is false, a simple check for higher versions than
+// version+patchlevel+sublevel is done.
+func kernelSupportsQuota(release string) bool {
+	type kernelSupport struct {
+		version      int
+		patchlevel   int
+		sublevel     int
+		extraversion int    // prefix of the part after the first "-"
+		distribution string // component of full extraversion
+		backport     bool   // backports have a fixed version/patchlevel/sublevel
+	}
+
+	quotaSupport := []kernelSupport{
+		{4, 17, 0, 0, "", false},       // standard 4.17+ versions
+		{3, 10, 0, 1062, ".el7", true}, // RHEL-7.7
+	}
+
+	vers := strings.Split(strings.SplitN(release, "-", 2)[0], ".")
+	version, err := strconv.Atoi(vers[0])
+	if err != nil {
+		klog.Errorf("failed to parse version from %s: %v", release, err)
+		return false
+	}
+	patchlevel, err := strconv.Atoi(vers[1])
+	if err != nil {
+		klog.Errorf("failed to parse patchlevel from %s: %v", release, err)
+		return false
+	}
+	sublevel := 0
+	if len(vers) >= 3 {
+		sublevel, err = strconv.Atoi(vers[2])
+		if err != nil {
+			klog.Errorf("failed to parse sublevel from %s: %v", release, err)
+			return false
+		}
+	}
+	extra := strings.SplitN(release, "-", 2)
+	extraversion := 0
+	if len(extra) == 2 {
+		// ignore errors, 1st component of extraversion does not need to be an int
+		extraversion, err = strconv.Atoi(strings.Split(extra[1], ".")[0])
+		if err != nil {
+			// "go lint" wants err to be checked...
+			extraversion = 0
+		}
+	}
+
+	// compare running kernel against known versions
+	for _, kernel := range quotaSupport {
+		if !kernel.backport {
+			// deal with the default case(s), find >= match for version, patchlevel, sublevel
+			if version > kernel.version || (version == kernel.version && patchlevel > kernel.patchlevel) ||
+				(version == kernel.version && patchlevel == kernel.patchlevel && sublevel >= kernel.sublevel) {
+				return true
+			}
+		} else {
+			// specific backport, match distribution initially
+			if !strings.Contains(release, kernel.distribution) {
+				continue
+			}
+
+			// strict match version, patchlevel, sublevel, and >= match extraversion
+			if version == kernel.version && patchlevel == kernel.patchlevel &&
+				sublevel == kernel.sublevel && extraversion >= kernel.extraversion {
+				return true
+			}
+		}
+	}
+	klog.Errorf("kernel %s does not support quota", release)
+	return false
+}
+
+// Load available ceph mounters installed on system into availableMounters
+// Called from driver.go's Run()
+func loadAvailableMounters(conf *util.Config) error {
+	// #nosec
+	fuseMounterProbe := exec.Command("ceph-fuse", "--version")
+	// #nosec
+	kernelMounterProbe := exec.Command("mount.ceph")
+
+	err := kernelMounterProbe.Run()
+	if err != nil {
+		klog.Errorf("failed to run mount.ceph %v", err)
+	} else {
+		// fetch the current running kernel info
+		utsname := unix.Utsname{}
+		err = unix.Uname(&utsname)
+		if err != nil {
+			return err
+		}
+		release := string(utsname.Release[:64])
+
+		if conf.ForceKernelCephFS || kernelSupportsQuota(release) {
+			klog.V(1).Infof("loaded mounter: %s", volumeMounterKernel)
+			availableMounters = append(availableMounters, volumeMounterKernel)
+		} else {
+			klog.V(1).Infof("kernel version < 4.17 might not support quota feature, hence not loading kernel client")
+		}
+	}
+
+	err = fuseMounterProbe.Run()
+	if err != nil {
+		klog.Errorf("failed to run ceph-fuse %v", err)
+	} else {
+		klog.V(1).Infof("loaded mounter: %s", volumeMounterFuse)
+		availableMounters = append(availableMounters, volumeMounterFuse)
+	}
+
+	if len(availableMounters) == 0 {
+		return errors.New("no ceph mounters found on system")
+	}
+
+	return nil
+}
+
+type volumeMounter interface {
+	mount(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error
+	name() string
+}
+
+func newMounter(volOptions *volumeOptions) (volumeMounter, error) {
+	// Get the mounter from the configuration
+
+	wantMounter := volOptions.Mounter
+
+	// Verify that it's available
+
+	var chosenMounter string
+
+	for _, availMounter := range availableMounters {
+		if availMounter == wantMounter {
+			chosenMounter = wantMounter
+			break
+		}
+	}
+
+	if chosenMounter == "" {
+		// Otherwise pick whatever is left
+		chosenMounter = availableMounters[0]
+		klog.V(4).Infof("requested mounter: %s, chosen mounter: %s", wantMounter, chosenMounter)
+	}
+
+	// Create the mounter
+
+	switch chosenMounter {
+	case volumeMounterFuse:
+		return &fuseMounter{}, nil
+	case volumeMounterKernel:
+		return &kernelMounter{}, nil
+	}
+
+	return nil, fmt.Errorf("unknown mounter '%s'", chosenMounter)
+}
+
+type fuseMounter struct{}
+
+func mountFuse(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
+	args := []string{
+		mountPoint,
+		"-m", volOptions.Monitors,
+		"-c", util.CephConfigPath,
+		"-n", cephEntityClientPrefix + cr.ID, "--keyfile=" + cr.KeyFile,
+		"-r", volOptions.RootPath,
+		"-o", "nonempty",
+	}
+
+	if volOptions.FuseMountOptions != "" {
+		args = append(args, ","+volOptions.FuseMountOptions)
+	}
+
+	if volOptions.FsName != "" {
+		args = append(args, "--client_mds_namespace="+volOptions.FsName)
+	}
+
+	_, stderr, err := execCommand(ctx, "ceph-fuse", args[:]...)
+	if err != nil {
+		return err
+	}
+
+	// Parse the output:
+	// We need "starting fuse" meaning the mount is ok
+	// and PID of the ceph-fuse daemon for unmount
+
+	match := fusePidRx.FindSubmatch(stderr)
+	if len(match) != 2 {
+		return fmt.Errorf("ceph-fuse failed: %s", stderr)
+	}
+
+	pid, err := strconv.Atoi(string(match[1]))
+	if err != nil {
+		return fmt.Errorf("failed to parse FUSE daemon PID: %v", err)
+	}
+
+	fusePidMapMtx.Lock()
+	fusePidMap[mountPoint] = pid
+	fusePidMapMtx.Unlock()
+
+	return nil
+}
+
+func (m *fuseMounter) mount(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
+	if err := util.CreateMountPoint(mountPoint); err != nil {
+		return err
+	}
+
+	return mountFuse(ctx, mountPoint, cr, volOptions)
+}
+
+func (m *fuseMounter) name() string { return "Ceph FUSE driver" }
+
+type kernelMounter struct{}
+
+func mountKernel(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
+	if err := execCommandErr(ctx, "modprobe", "ceph"); err != nil {
+		return err
+	}
+
+	args := []string{
+		"-t", "ceph",
+		fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
+		mountPoint,
+	}
+	optionsStr := fmt.Sprintf("name=%s,secretfile=%s", cr.ID, cr.KeyFile)
+	if volOptions.FsName != "" {
+		optionsStr += fmt.Sprintf(",mds_namespace=%s", volOptions.FsName)
+	}
+	if volOptions.KernelMountOptions != "" {
+		optionsStr += fmt.Sprintf(",%s", volOptions.KernelMountOptions)
+	}
+
+	if !strings.Contains(volOptions.KernelMountOptions, netDev) {
+		optionsStr += fmt.Sprintf(",%s", netDev)
+	}
+
+	args = append(args, "-o", optionsStr)
+
+	return execCommandErr(ctx, "mount", args[:]...)
+}
+
+func (m *kernelMounter) mount(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
+	if err := util.CreateMountPoint(mountPoint); err != nil {
+		return err
+	}
+
+	return mountKernel(ctx, mountPoint, cr, volOptions)
+}
+
+func (m *kernelMounter) name() string { return "Ceph kernel client" }
+
+func bindMount(ctx context.Context, from, to string, readOnly bool, mntOptions []string) error {
+	mntOptionSli := strings.Join(mntOptions, ",")
+	if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, from, to); err != nil {
+		return fmt.Errorf("failed to bind-mount %s to %s: %v", from, to, err)
+	}
+
+	if readOnly {
+		mntOptionSli += ",remount"
+		if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, to); err != nil {
+			return fmt.Errorf("failed read-only remount of %s: %v", to, err)
+		}
+	}
+
+	return nil
+}
+
+func unmountVolume(ctx context.Context, mountPoint string) error {
+	if err := execCommandErr(ctx, "umount", mountPoint); err != nil {
+		if strings.Contains(err.Error(), fmt.Sprintf("exit status 32: umount: %s: not mounted", mountPoint)) ||
+			strings.Contains(err.Error(), "No such file or directory") {
+			return nil
+		}
+		return err
+	}
+
+	fusePidMapMtx.Lock()
+	pid, ok := fusePidMap[mountPoint]
+	if ok {
+		delete(fusePidMap, mountPoint)
+	}
+	fusePidMapMtx.Unlock()
+
+	if ok {
+		p, err := os.FindProcess(pid)
+		if err != nil {
+			klog.Warningf(util.Log(ctx, "failed to find process %d: %v"), pid, err)
+		} else {
+			if _, err = p.Wait(); err != nil {
+				klog.Warningf(util.Log(ctx, "%d is not a child process: %v"), pid, err)
+			}
+		}
+	}
+
+	return nil
+}

internal/cephfs/volumemounter_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"testing"
+)
+
+func init() {
+}
+
+func TestKernelSupportsQuota(t *testing.T) {
+	supportsQuota := []string{
+		"4.17.0",
+		"5.0.0",
+		"4.17.0-rc1",
+		"4.18.0-80.el8",
+		"3.10.0-1062.el7.x86_64",     // 1st backport
+		"3.10.0-1062.4.1.el7.x86_64", // updated backport
+	}
+
+	noQuota := []string{
+		"2.6.32-754.15.3.el6.x86_64", // too old
+		"3.10.0-123.el7.x86_64",      // too old for backport
+		"3.10.0-1062.4.1.el8.x86_64", // nonexisting RHEL-8 kernel
+		"3.11.0-123.el7.x86_64",      // nonexisting RHEL-7 kernel
+	}
+
+	for _, kernel := range supportsQuota {
+		ok := kernelSupportsQuota(kernel)
+		if !ok {
+			t.Errorf("support expected for %s", kernel)
+		}
+	}
+
+	for _, kernel := range noQuota {
+		ok := kernelSupportsQuota(kernel)
+		if ok {
+			t.Errorf("no support expected for %s", kernel)
+		}
+	}
+}

internal/cephfs/volumeoptions.go

@@ -0,0 +1,382 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/pkg/errors"
+
+	"github.com/ceph/ceph-csi/internal/util"
+)
+
+type volumeOptions struct {
+	TopologyPools       *[]util.TopologyConstrainedPool
+	TopologyRequirement *csi.TopologyRequirement
+	Topology            map[string]string
+	RequestName         string
+	NamePrefix          string
+	Size                int64
+	ClusterID           string
+	FsName              string
+	FscID               int64
+	MetadataPool        string
+	Monitors            string `json:"monitors"`
+	Pool                string `json:"pool"`
+	RootPath            string `json:"rootPath"`
+	Mounter             string `json:"mounter"`
+	ProvisionVolume     bool   `json:"provisionVolume"`
+	KernelMountOptions  string `json:"kernelMountOptions"`
+	FuseMountOptions    string `json:"fuseMountOptions"`
+}
+
+func validateNonEmptyField(field, fieldName string) error {
+	if field == "" {
+		return fmt.Errorf("parameter '%s' cannot be empty", fieldName)
+	}
+
+	return nil
+}
+
+func extractOptionalOption(dest *string, optionLabel string, options map[string]string) error {
+	opt, ok := options[optionLabel]
+	if !ok {
+		// Option not found, no error as it is optional
+		return nil
+	}
+
+	if err := validateNonEmptyField(opt, optionLabel); err != nil {
+		return err
+	}
+
+	*dest = opt
+	return nil
+}
+
+func extractOption(dest *string, optionLabel string, options map[string]string) error {
+	opt, ok := options[optionLabel]
+	if !ok {
+		return fmt.Errorf("missing required field %s", optionLabel)
+	}
+
+	if err := validateNonEmptyField(opt, optionLabel); err != nil {
+		return err
+	}
+
+	*dest = opt
+	return nil
+}
+
+func validateMounter(m string) error {
+	switch m {
+	case volumeMounterFuse:
+	case volumeMounterKernel:
+	default:
+		return fmt.Errorf("unknown mounter '%s'. Valid options are 'fuse' and 'kernel'", m)
+	}
+
+	return nil
+}
+
+func extractMounter(dest *string, options map[string]string) error {
+	if err := extractOptionalOption(dest, "mounter", options); err != nil {
+		return err
+	}
+
+	if *dest != "" {
+		if err := validateMounter(*dest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func getMonsAndClusterID(options map[string]string) (string, string, error) {
+	clusterID, ok := options["clusterID"]
+	if !ok {
+		err := fmt.Errorf("clusterID must be set")
+		return "", "", err
+	}
+
+	if err := validateNonEmptyField(clusterID, "clusterID"); err != nil {
+		return "", "", err
+	}
+
+	monitors, err := util.Mons(csiConfigFile, clusterID)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to fetch monitor list using clusterID (%s)", clusterID)
+		return "", "", err
+	}
+
+	return monitors, clusterID, err
+}
+
+// newVolumeOptions generates a new instance of volumeOptions from the provided
+// CSI request parameters
+func newVolumeOptions(ctx context.Context, requestName string, req *csi.CreateVolumeRequest,
+	secret map[string]string) (*volumeOptions, error) {
+	var (
+		opts volumeOptions
+		err  error
+	)
+
+	volOptions := req.GetParameters()
+
+	opts.Monitors, opts.ClusterID, err = getMonsAndClusterID(volOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = extractOptionalOption(&opts.Pool, "pool", volOptions); err != nil {
+		return nil, err
+	}
+
+	if err = extractMounter(&opts.Mounter, volOptions); err != nil {
+		return nil, err
+	}
+
+	if err = extractOption(&opts.FsName, "fsName", volOptions); err != nil {
+		return nil, err
+	}
+
+	if err = extractOptionalOption(&opts.KernelMountOptions, "kernelMountOptions", volOptions); err != nil {
+		return nil, err
+	}
+
+	if err = extractOptionalOption(&opts.FuseMountOptions, "fuseMountOptions", volOptions); err != nil {
+		return nil, err
+	}
+
+	opts.RequestName = requestName
+
+	cr, err := util.NewAdminCredentials(secret)
+	if err != nil {
+		return nil, err
+	}
+	defer cr.DeleteCredentials()
+
+	opts.FscID, err = getFscID(ctx, opts.Monitors, cr, opts.FsName)
+	if err != nil {
+		return nil, err
+	}
+
+	opts.MetadataPool, err = getMetadataPool(ctx, opts.Monitors, cr, opts.FsName)
+	if err != nil {
+		return nil, err
+	}
+
+	// store topology information from the request
+	opts.TopologyPools, opts.TopologyRequirement, err = util.GetTopologyFromRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: we need an API to fetch subvolume attributes (size/datapool and others), based
+	// on which we can evaluate which topology this belongs to.
+	// CephFS tracker: https://tracker.ceph.com/issues/44277
+	if opts.TopologyPools != nil {
+		return nil, errors.New("topology based provisioning is not supported for CephFS backed volumes")
+	}
+
+	opts.ProvisionVolume = true
+
+	return &opts, nil
+}
+
+// newVolumeOptionsFromVolID generates a new instance of volumeOptions and volumeIdentifier
+// from the provided CSI VolumeID
+func newVolumeOptionsFromVolID(ctx context.Context, volID string, volOpt, secrets map[string]string) (*volumeOptions, *volumeIdentifier, error) {
+	var (
+		vi         util.CSIIdentifier
+		volOptions volumeOptions
+		vid        volumeIdentifier
+	)
+
+	// Decode the VolID first, to detect older volumes or pre-provisioned volumes
+	// before other errors
+	err := vi.DecomposeCSIID(volID)
+	if err != nil {
+		err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
+		return nil, nil, ErrInvalidVolID{err}
+	}
+	volOptions.ClusterID = vi.ClusterID
+	vid.VolumeID = volID
+	volOptions.FscID = vi.LocationID
+
+	if volOptions.Monitors, err = util.Mons(csiConfigFile, vi.ClusterID); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to fetch monitor list using clusterID (%s)", vi.ClusterID)
+	}
+
+	cr, err := util.NewAdminCredentials(secrets)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer cr.DeleteCredentials()
+
+	volOptions.FsName, err = getFsName(ctx, volOptions.Monitors, cr, volOptions.FscID)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	volOptions.MetadataPool, err = getMetadataPool(ctx, volOptions.Monitors, cr, volOptions.FsName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	imageAttributes, err := volJournal.GetImageAttributes(ctx, volOptions.Monitors, cr,
+		volOptions.MetadataPool, vi.ObjectUUID, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	volOptions.RequestName = imageAttributes.RequestName
+	vid.FsSubvolName = imageAttributes.ImageName
+
+	if volOpt != nil {
+		if err = extractOptionalOption(&volOptions.Pool, "pool", volOpt); err != nil {
+			return nil, nil, err
+		}
+
+		if err = extractOptionalOption(&volOptions.KernelMountOptions, "kernelMountOptions", volOpt); err != nil {
+			return nil, nil, err
+		}
+
+		if err = extractOptionalOption(&volOptions.FuseMountOptions, "fuseMountOptions", volOpt); err != nil {
+			return nil, nil, err
+		}
+
+		if err = extractMounter(&volOptions.Mounter, volOpt); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	volOptions.ProvisionVolume = true
+
+	volOptions.RootPath, err = getVolumeRootPathCeph(ctx, &volOptions, cr, volumeID(vid.FsSubvolName))
+	if err != nil {
+		return &volOptions, &vid, err
+	}
+
+	return &volOptions, &vid, nil
+}
+
+// newVolumeOptionsFromVersion1Context generates a new instance of volumeOptions and
+// volumeIdentifier from the provided CSI volume context, if the provided context was
+// for a volume created by version 1.0.0 (or prior) of the CSI plugin
+func newVolumeOptionsFromVersion1Context(volID string, options, secrets map[string]string) (*volumeOptions, *volumeIdentifier, error) {
+	var (
+		opts                volumeOptions
+		vid                 volumeIdentifier
+		provisionVolumeBool string
+		err                 error
+	)
+
+	// Check if monitors is part of the options, that is an indicator this is an 1.0.0 volume
+	if err = extractOption(&opts.Monitors, "monitors", options); err != nil {
+		return nil, nil, err
+	}
+
+	// check if there are mon values in secret and if so override option retrieved monitors from
+	// monitors in the secret
+	mon, err := util.GetMonValFromSecret(secrets)
+	if err == nil && len(mon) > 0 {
+		opts.Monitors = mon
+	}
+
+	if err = extractOption(&provisionVolumeBool, "provisionVolume", options); err != nil {
+		return nil, nil, err
+	}
+
+	if opts.ProvisionVolume, err = strconv.ParseBool(provisionVolumeBool); err != nil {
+		return nil, nil, fmt.Errorf("failed to parse provisionVolume: %v", err)
+	}
+
+	if opts.ProvisionVolume {
+		if err = extractOption(&opts.Pool, "pool", options); err != nil {
+			return nil, nil, err
+		}
+
+		opts.RootPath = getVolumeRootPathCephDeprecated(volumeID(volID))
+	} else {
+		if err = extractOption(&opts.RootPath, "rootPath", options); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if err = extractMounter(&opts.Mounter, options); err != nil {
+		return nil, nil, err
+	}
+
+	vid.FsSubvolName = volID
+	vid.VolumeID = volID
+
+	return &opts, &vid, nil
+}
+
+// newVolumeOptionsFromStaticVolume generates a new instance of volumeOptions and
+// volumeIdentifier from the provided CSI volume context, if the provided context is
+// detected to be a statically provisioned volume
+func newVolumeOptionsFromStaticVolume(volID string, options map[string]string) (*volumeOptions, *volumeIdentifier, error) {
+	var (
+		opts      volumeOptions
+		vid       volumeIdentifier
+		staticVol bool
+		err       error
+	)
+
+	val, ok := options["staticVolume"]
+	if !ok {
+		return nil, nil, ErrNonStaticVolume{err}
+	}
+
+	if staticVol, err = strconv.ParseBool(val); err != nil {
+		return nil, nil, fmt.Errorf("failed to parse preProvisionedVolume: %v", err)
+	}
+
+	if !staticVol {
+		return nil, nil, ErrNonStaticVolume{err}
+	}
+
+	// Volume is static, and ProvisionVolume carries bool stating if it was provisioned, hence
+	// store NOT of static boolean
+	opts.ProvisionVolume = !staticVol
+
+	opts.Monitors, opts.ClusterID, err = getMonsAndClusterID(options)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if err = extractOption(&opts.RootPath, "rootPath", options); err != nil {
+		return nil, nil, err
+	}
+
+	if err = extractOption(&opts.FsName, "fsName", options); err != nil {
+		return nil, nil, err
+	}
+
+	if err = extractMounter(&opts.Mounter, options); err != nil {
+		return nil, nil, err
+	}
+
+	vid.FsSubvolName = opts.RootPath
+	vid.VolumeID = volID
+
+	return &opts, &vid, nil
+}

internal/csi-common/controllerserver-default.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"context"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+// DefaultControllerServer points to default driver
+type DefaultControllerServer struct {
+	Driver *CSIDriver
+}
+
+// ControllerPublishVolume publish volume on node
+func (cs *DefaultControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ControllerUnpublishVolume unpublish on node
+func (cs *DefaultControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ControllerExpandVolume expand volume
+func (cs *DefaultControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ListVolumes lists volumes
+func (cs *DefaultControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// GetCapacity get volume capacity
+func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ControllerGetCapabilities implements the default GRPC callout.
+// Default supports all capabilities
+func (cs *DefaultControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
+	klog.V(5).Infof(util.Log(ctx, "Using default ControllerGetCapabilities"))
+
+	return &csi.ControllerGetCapabilitiesResponse{
+		Capabilities: cs.Driver.cap,
+	}, nil
+}
+
+// CreateSnapshot creates snapshot
+func (cs *DefaultControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// DeleteSnapshot deletes snapshot
+func (cs *DefaultControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ListSnapshots lists snapshots
+func (cs *DefaultControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}

internal/csi-common/driver.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"fmt"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+// CSIDriver stores driver information
+type CSIDriver struct {
+	name    string
+	nodeID  string
+	version string
+	// topology constraints that this nodeserver will advertise
+	topology map[string]string
+	cap      []*csi.ControllerServiceCapability
+	vc       []*csi.VolumeCapability_AccessMode
+}
+
+// NewCSIDriver Creates a NewCSIDriver object. Assumes vendor
+// version is equal to driver version &  does not support optional
+// driver plugin info manifest field. Refer to CSI spec for more details.
+func NewCSIDriver(name, v, nodeID string) *CSIDriver {
+	if name == "" {
+		klog.Errorf("Driver name missing")
+		return nil
+	}
+
+	if nodeID == "" {
+		klog.Errorf("NodeID missing")
+		return nil
+	}
+	// TODO version format and validation
+	if v == "" {
+		klog.Errorf("Version argument missing")
+		return nil
+	}
+
+	driver := CSIDriver{
+		name:    name,
+		version: v,
+		nodeID:  nodeID,
+	}
+
+	return &driver
+}
+
+// ValidateControllerServiceRequest validates the controller
+// plugin capabilities
+func (d *CSIDriver) ValidateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error {
+	if c == csi.ControllerServiceCapability_RPC_UNKNOWN {
+		return nil
+	}
+
+	for _, cap := range d.cap {
+		if c == cap.GetRpc().GetType() {
+			return nil
+		}
+	}
+	return status.Error(codes.InvalidArgument, fmt.Sprintf("%s", c)) //nolint
+}
+
+// AddControllerServiceCapabilities stores the controller capabilities
+// in driver object
+func (d *CSIDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) {
+	var csc []*csi.ControllerServiceCapability
+
+	for _, c := range cl {
+		klog.V(1).Infof("Enabling controller service capability: %v", c.String())
+		csc = append(csc, NewControllerServiceCapability(c))
+	}
+
+	d.cap = csc
+}
+
+// AddVolumeCapabilityAccessModes stores volume access modes
+func (d *CSIDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode {
+	var vca []*csi.VolumeCapability_AccessMode
+	for _, c := range vc {
+		klog.V(1).Infof("Enabling volume access mode: %v", c.String())
+		vca = append(vca, NewVolumeCapabilityAccessMode(c))
+	}
+	d.vc = vca
+	return vca
+}
+
+// GetVolumeCapabilityAccessModes returns access modes
+func (d *CSIDriver) GetVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode {
+	return d.vc
+}

internal/csi-common/identityserver-default.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"context"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+// DefaultIdentityServer stores driver object
+type DefaultIdentityServer struct {
+	Driver *CSIDriver
+}
+
+// GetPluginInfo returns plugin information
+func (ids *DefaultIdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
+	klog.V(5).Infof(util.Log(ctx, "Using default GetPluginInfo"))
+
+	if ids.Driver.name == "" {
+		return nil, status.Error(codes.Unavailable, "Driver name not configured")
+	}
+
+	if ids.Driver.version == "" {
+		return nil, status.Error(codes.Unavailable, "Driver is missing version")
+	}
+
+	return &csi.GetPluginInfoResponse{
+		Name:          ids.Driver.name,
+		VendorVersion: ids.Driver.version,
+	}, nil
+}
+
+// Probe returns empty response
+func (ids *DefaultIdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) {
+	return &csi.ProbeResponse{}, nil
+}
+
+// GetPluginCapabilities returns plugin capabilities
+func (ids *DefaultIdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
+	klog.V(5).Infof(util.Log(ctx, "Using default capabilities"))
+	return &csi.GetPluginCapabilitiesResponse{
+		Capabilities: []*csi.PluginCapability{
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
+					},
+				},
+			},
+		},
+	}, nil
+}

internal/csi-common/nodeserver-default.go

@@ -0,0 +1,197 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"context"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+// DefaultNodeServer stores driver object
+type DefaultNodeServer struct {
+	Driver *CSIDriver
+	Type   string
+}
+
+// NodeStageVolume returns unimplemented response
+func (ns *DefaultNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// NodeUnstageVolume returns unimplemented response
+func (ns *DefaultNodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// NodeExpandVolume returns unimplemented response
+func (ns *DefaultNodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+// NodeGetInfo returns node ID
+func (ns *DefaultNodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
+	klog.V(5).Infof(util.Log(ctx, "Using default NodeGetInfo"))
+
+	csiTopology := &csi.Topology{
+		Segments: ns.Driver.topology,
+	}
+
+	return &csi.NodeGetInfoResponse{
+		NodeId:             ns.Driver.nodeID,
+		AccessibleTopology: csiTopology,
+	}, nil
+}
+
+// NodeGetCapabilities returns RPC unknow capability
+func (ns *DefaultNodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
+	klog.V(5).Infof(util.Log(ctx, "Using default NodeGetCapabilities"))
+
+	return &csi.NodeGetCapabilitiesResponse{
+		Capabilities: []*csi.NodeServiceCapability{
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_UNKNOWN,
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+// NodeGetVolumeStats returns volume stats
+func (ns *DefaultNodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
+	var err error
+	targetPath := req.GetVolumePath()
+	if targetPath == "" {
+		err = fmt.Errorf("targetpath %v is empty", targetPath)
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	/*
+		volID := req.GetVolumeId()
+
+		TODO: Map the volumeID to the targetpath.
+
+		CephFS:
+		   we need secret to connect to the ceph cluster to get the volumeID from volume
+		   Name, however `secret` field/option is not available  in NodeGetVolumeStats spec,
+		   Below issue covers this request and once its available, we can do the validation
+		   as per the spec.
+
+		   https://github.com/container-storage-interface/spec/issues/371
+
+		RBD:
+		   Below issue covers this request for RBD and once its available, we can do the validation
+		   as per the spec.
+
+		   https://github.com/ceph/ceph-csi/issues/511
+
+	*/
+
+	isMnt, err := util.IsMountPoint(targetPath)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, status.Errorf(codes.InvalidArgument, "targetpath %s doesnot exist", targetPath)
+		}
+		return nil, err
+	}
+	if !isMnt {
+		return nil, status.Errorf(codes.InvalidArgument, "targetpath %s is not mounted", targetPath)
+	}
+
+	cephMetricsProvider := volume.NewMetricsStatFS(targetPath)
+	volMetrics, volMetErr := cephMetricsProvider.GetMetrics()
+	if volMetErr != nil {
+		return nil, status.Error(codes.Internal, volMetErr.Error())
+	}
+
+	available, ok := (*(volMetrics.Available)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch available bytes"))
+	}
+	capacity, ok := (*(volMetrics.Capacity)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch capacity bytes"))
+		return nil, status.Error(codes.Unknown, "failed to fetch capacity bytes")
+	}
+	used, ok := (*(volMetrics.Used)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch used bytes"))
+	}
+	inodes, ok := (*(volMetrics.Inodes)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch available inodes"))
+		return nil, status.Error(codes.Unknown, "failed to fetch available inodes")
+	}
+	inodesFree, ok := (*(volMetrics.InodesFree)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch free inodes"))
+	}
+
+	inodesUsed, ok := (*(volMetrics.InodesUsed)).AsInt64()
+	if !ok {
+		klog.Errorf(util.Log(ctx, "failed to fetch used inodes"))
+	}
+	return &csi.NodeGetVolumeStatsResponse{
+		Usage: []*csi.VolumeUsage{
+			{
+				Available: available,
+				Total:     capacity,
+				Used:      used,
+				Unit:      csipbv1.VolumeUsage_BYTES,
+			},
+			{
+				Available: inodesFree,
+				Total:     inodes,
+				Used:      inodesUsed,
+				Unit:      csipbv1.VolumeUsage_INODES,
+			},
+		},
+	}, nil
+}
+
+// ConstructMountOptions returns only unique mount options in slice
+func ConstructMountOptions(mountOptions []string, volCap *csi.VolumeCapability) []string {
+	if m := volCap.GetMount(); m != nil {
+		hasOption := func(options []string, opt string) bool {
+			for _, o := range options {
+				if o == opt {
+					return true
+				}
+			}
+			return false
+		}
+		for _, f := range m.MountFlags {
+			if !hasOption(mountOptions, f) {
+				mountOptions = append(mountOptions, f)
+			}
+		}
+	}
+	return mountOptions
+}

internal/csi-common/server.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc"
+	"k8s.io/klog"
+)
+
+// NonBlockingGRPCServer defines Non blocking GRPC server interfaces
+type NonBlockingGRPCServer interface {
+	// Start services at the endpoint
+	Start(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool)
+	// Waits for the service to stop
+	Wait()
+	// Stops the service gracefully
+	Stop()
+	// Stops the service forcefully
+	ForceStop()
+}
+
+// NewNonBlockingGRPCServer return non-blocking GRPC
+func NewNonBlockingGRPCServer() NonBlockingGRPCServer {
+	return &nonBlockingGRPCServer{}
+}
+
+// NonBlocking server
+type nonBlockingGRPCServer struct {
+	wg     sync.WaitGroup
+	server *grpc.Server
+}
+
+// Start start service on endpoint
+func (s *nonBlockingGRPCServer) Start(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool) {
+	s.wg.Add(1)
+	go s.serve(endpoint, hstOptions, ids, cs, ns, metrics)
+}
+
+// Wait blocks until the WaitGroup counter
+func (s *nonBlockingGRPCServer) Wait() {
+	s.wg.Wait()
+}
+
+// GracefulStop stops the gRPC server gracefully.
+func (s *nonBlockingGRPCServer) Stop() {
+	s.server.GracefulStop()
+}
+
+// Stop stops the gRPC server.
+func (s *nonBlockingGRPCServer) ForceStop() {
+	s.server.Stop()
+}
+
+func (s *nonBlockingGRPCServer) serve(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool) {
+	proto, addr, err := parseEndpoint(endpoint)
+	if err != nil {
+		klog.Fatal(err.Error())
+	}
+
+	if proto == "unix" {
+		addr = "/" + addr
+		if e := os.Remove(addr); e != nil && !os.IsNotExist(e) {
+			klog.Fatalf("Failed to remove %s, error: %s", addr, e.Error())
+		}
+	}
+
+	listener, err := net.Listen(proto, addr)
+	if err != nil {
+		klog.Fatalf("Failed to listen: %v", err)
+	}
+
+	middleWare := []grpc.UnaryServerInterceptor{contextIDInjector, logGRPC, panicHandler}
+	if metrics {
+		middleWare = append(middleWare, grpc_prometheus.UnaryServerInterceptor)
+	}
+	opts := []grpc.ServerOption{
+		grpc_middleware.WithUnaryServerChain(middleWare...),
+	}
+
+	server := grpc.NewServer(opts...)
+	s.server = server
+
+	if ids != nil {
+		csi.RegisterIdentityServer(server, ids)
+	}
+	if cs != nil {
+		csi.RegisterControllerServer(server, cs)
+	}
+	if ns != nil {
+		csi.RegisterNodeServer(server, ns)
+	}
+	klog.V(1).Infof("Listening for connections on address: %#v", listener.Addr())
+	if metrics {
+		ho := strings.Split(hstOptions, ",")
+		if len(ho) != 3 {
+			klog.Fatalf("invalid histogram options provided: %v", hstOptions)
+		}
+		start, e := strconv.ParseFloat(ho[0], 32)
+		if e != nil {
+			klog.Fatalf("failed to parse histogram start value: %v", e)
+		}
+		factor, e := strconv.ParseFloat(ho[1], 32)
+		if err != nil {
+			klog.Fatalf("failed to parse histogram factor value: %v", e)
+		}
+		count, e := strconv.Atoi(ho[2])
+		if err != nil {
+			klog.Fatalf("failed to parse histogram count value: %v", e)
+		}
+		buckets := prometheus.ExponentialBuckets(start, factor, count)
+		bktOptios := grpc_prometheus.WithHistogramBuckets(buckets)
+		grpc_prometheus.EnableHandlingTimeHistogram(bktOptios)
+		grpc_prometheus.Register(server)
+	}
+	err = server.Serve(listener)
+	if err != nil {
+		klog.Fatalf("Failed to server: %v", err)
+	}
+}

internal/csi-common/utils.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"context"
+	"fmt"
+	"runtime/debug"
+	"strings"
+	"sync/atomic"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+func parseEndpoint(ep string) (string, string, error) {
+	if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
+		s := strings.SplitN(ep, "://", 2)
+		if s[1] != "" {
+			return s[0], s[1], nil
+		}
+	}
+	return "", "", fmt.Errorf("invalid endpoint: %v", ep)
+}
+
+// NewVolumeCapabilityAccessMode returns volume access mode
+func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode {
+	return &csi.VolumeCapability_AccessMode{Mode: mode}
+}
+
+// NewDefaultNodeServer initializes default node server
+func NewDefaultNodeServer(d *CSIDriver, t string, topology map[string]string) *DefaultNodeServer {
+	d.topology = topology
+	return &DefaultNodeServer{
+		Driver: d,
+		Type:   t,
+	}
+}
+
+// NewDefaultIdentityServer initializes default identity servier
+func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer {
+	return &DefaultIdentityServer{
+		Driver: d,
+	}
+}
+
+// NewDefaultControllerServer initializes default controller server
+func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer {
+	return &DefaultControllerServer{
+		Driver: d,
+	}
+}
+
+// NewControllerServiceCapability returns controller capabilities
+func NewControllerServiceCapability(ctrlCap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability {
+	return &csi.ControllerServiceCapability{
+		Type: &csi.ControllerServiceCapability_Rpc{
+			Rpc: &csi.ControllerServiceCapability_RPC{
+				Type: ctrlCap,
+			},
+		},
+	}
+}
+
+// RunNodePublishServer starts node server
+func RunNodePublishServer(endpoint, hstOption string, d *CSIDriver, ns csi.NodeServer, m bool) {
+	ids := NewDefaultIdentityServer(d)
+
+	s := NewNonBlockingGRPCServer()
+	s.Start(endpoint, hstOption, ids, nil, ns, m)
+	s.Wait()
+}
+
+// RunControllerPublishServer starts controller server
+func RunControllerPublishServer(endpoint, hstOption string, d *CSIDriver, cs csi.ControllerServer, m bool) {
+	ids := NewDefaultIdentityServer(d)
+
+	s := NewNonBlockingGRPCServer()
+	s.Start(endpoint, hstOption, ids, cs, nil, m)
+	s.Wait()
+}
+
+// RunControllerandNodePublishServer starts both controller and node server
+func RunControllerandNodePublishServer(endpoint, hstOption string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer, m bool) {
+	ids := NewDefaultIdentityServer(d)
+
+	s := NewNonBlockingGRPCServer()
+	s.Start(endpoint, hstOption, ids, cs, ns, m)
+	s.Wait()
+}
+
+func getReqID(req interface{}) string {
+	// if req is nil empty string will be returned
+	reqID := ""
+	switch r := req.(type) {
+	case *csi.CreateVolumeRequest:
+		reqID = r.Name
+
+	case *csi.DeleteVolumeRequest:
+		reqID = r.VolumeId
+
+	case *csi.CreateSnapshotRequest:
+		reqID = r.Name
+	case *csi.DeleteSnapshotRequest:
+		reqID = r.SnapshotId
+
+	case *csi.ControllerExpandVolumeRequest:
+		reqID = r.VolumeId
+
+	case *csi.NodeStageVolumeRequest:
+		reqID = r.VolumeId
+	case *csi.NodeUnstageVolumeRequest:
+		reqID = r.VolumeId
+
+	case *csi.NodePublishVolumeRequest:
+		reqID = r.VolumeId
+	case *csi.NodeUnpublishVolumeRequest:
+		reqID = r.VolumeId
+
+	case *csi.NodeExpandVolumeRequest:
+		reqID = r.VolumeId
+	}
+	return reqID
+}
+
+var id uint64
+
+func contextIDInjector(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+	atomic.AddUint64(&id, 1)
+	ctx = context.WithValue(ctx, util.CtxKey, id)
+	reqID := getReqID(req)
+	if reqID != "" {
+		ctx = context.WithValue(ctx, util.ReqID, reqID)
+	}
+	return handler(ctx, req)
+}
+
+func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	klog.V(3).Infof(util.Log(ctx, "GRPC call: %s"), info.FullMethod)
+	klog.V(5).Infof(util.Log(ctx, "GRPC request: %s"), protosanitizer.StripSecrets(req))
+	resp, err := handler(ctx, req)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "GRPC error: %v"), err)
+	} else {
+		klog.V(5).Infof(util.Log(ctx, "GRPC response: %s"), protosanitizer.StripSecrets(resp))
+	}
+	return resp, err
+}
+
+func panicHandler(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			klog.Errorf("panic occurred: %v", r)
+			debug.PrintStack()
+			err = status.Errorf(codes.Internal, "panic %v", r)
+		}
+	}()
+	return handler(ctx, req)
+}

internal/csi-common/utils_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2019 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csicommon
+
+import (
+	"testing"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+var fakeID = "fake-id"
+
+func TestGetReqID(t *testing.T) {
+	req := []interface{}{
+		&csi.CreateVolumeRequest{
+			Name: fakeID,
+		},
+		&csi.DeleteVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.CreateSnapshotRequest{
+			Name: fakeID,
+		},
+		&csi.DeleteSnapshotRequest{
+			SnapshotId: fakeID,
+		},
+
+		&csi.ControllerExpandVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.NodeStageVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.NodeUnstageVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.NodePublishVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.NodeUnpublishVolumeRequest{
+			VolumeId: fakeID,
+		},
+		&csi.NodeExpandVolumeRequest{
+			VolumeId: fakeID,
+		},
+	}
+	for _, r := range req {
+		if got := getReqID(r); got != fakeID {
+			t.Errorf("getReqID() = %v, want %v", got, fakeID)
+		}
+	}
+
+	// test for nil request
+	if got := getReqID(nil); got != "" {
+		t.Errorf("getReqID() = %v, want empty string", got)
+	}
+}

internal/liveness/liveness.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package liveness
+
+import (
+	"context"
+	"time"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	connlib "github.com/kubernetes-csi/csi-lib-utils/connection"
+	"github.com/kubernetes-csi/csi-lib-utils/metrics"
+	"github.com/kubernetes-csi/csi-lib-utils/rpc"
+	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc"
+	"k8s.io/klog"
+)
+
+var (
+	liveness = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "csi",
+		Name:      "liveness",
+		Help:      "Liveness Probe",
+	})
+)
+
+func getLiveness(timeout time.Duration, csiConn *grpc.ClientConn) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	klog.V(5).Info("Sending probe request to CSI driver")
+	ready, err := rpc.Probe(ctx, csiConn)
+	if err != nil {
+		liveness.Set(0)
+		klog.Errorf("health check failed: %v", err)
+		return
+	}
+
+	if !ready {
+		liveness.Set(0)
+		klog.Error("driver responded but is not ready")
+		return
+	}
+	liveness.Set(1)
+	klog.V(3).Infof("Health check succeeded")
+}
+
+func recordLiveness(endpoint, drivername string, pollTime, timeout time.Duration) {
+	liveMetricsManager := metrics.NewCSIMetricsManager(drivername)
+	// register prometheus metrics
+	err := prometheus.Register(liveness)
+	if err != nil {
+		klog.Fatalln(err)
+	}
+
+	csiConn, err := connlib.Connect(endpoint, liveMetricsManager)
+	if err != nil {
+		// connlib should retry forever so a returned error should mean
+		// the grpc client is misconfigured rather than an error on the network
+		klog.Fatalf("failed to establish connection to CSI driver: %v", err)
+	}
+
+	// get liveness periodically
+	ticker := time.NewTicker(pollTime)
+	defer ticker.Stop()
+	for range ticker.C {
+		getLiveness(timeout, csiConn)
+	}
+}
+
+// Run starts liveness collection and prometheus endpoint
+func Run(conf *util.Config) {
+	klog.V(3).Infof("Liveness Running")
+
+	// start liveness collection
+	go recordLiveness(conf.Endpoint, conf.DriverName, conf.PollTime, conf.PoolTimeout)
+
+	// start up prometheus endpoint
+	util.StartMetricsServer(conf)
+}

internal/rbd/controllerserver.go

@@ -0,0 +1,818 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"fmt"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+)
+
+const (
+	oneGB = 1073741824
+)
+
+// ControllerServer struct of rbd CSI driver with supported methods of CSI
+// controller server spec.
+type ControllerServer struct {
+	*csicommon.DefaultControllerServer
+	MetadataStore util.CachePersister
+	// A map storing all volumes with ongoing operations so that additional operations
+	// for that same volume (as defined by VolumeID/volume name) return an Aborted error
+	VolumeLocks *util.VolumeLocks
+
+	// A map storing all volumes with ongoing operations so that additional operations
+	// for that same snapshot (as defined by SnapshotID/snapshot name) return an Aborted error
+	SnapshotLocks *util.VolumeLocks
+}
+
+func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.CreateVolumeRequest) error {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+		klog.Errorf(util.Log(ctx, "invalid create volume req: %v"), protosanitizer.StripSecrets(req))
+		return err
+	}
+	// Check sanity of request Name, Volume Capabilities
+	if req.Name == "" {
+		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
+	}
+	if req.VolumeCapabilities == nil {
+		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
+	}
+	options := req.GetParameters()
+	if value, ok := options["clusterID"]; !ok || value == "" {
+		return status.Error(codes.InvalidArgument, "missing or empty cluster ID to provision volume from")
+	}
+	if value, ok := options["pool"]; !ok || value == "" {
+		return status.Error(codes.InvalidArgument, "missing or empty pool name to provision volume from")
+	}
+
+	if value, ok := options["dataPool"]; ok && value == "" {
+		return status.Error(codes.InvalidArgument, "empty datapool name to provision volume from")
+	}
+	if value, ok := options["volumeNamePrefix"]; ok && value == "" {
+		return status.Error(codes.InvalidArgument, "empty volume name prefix to provision volume from")
+	}
+	return nil
+}
+
+func (cs *ControllerServer) parseVolCreateRequest(ctx context.Context, req *csi.CreateVolumeRequest) (*rbdVolume, error) {
+	// TODO (sbezverk) Last check for not exceeding total storage capacity
+
+	isMultiNode := false
+	isBlock := false
+	for _, cap := range req.VolumeCapabilities {
+		// RO modes need to be handled independently (ie right now even if access mode is RO, they'll be RW upon attach)
+		if cap.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER {
+			isMultiNode = true
+		}
+		if cap.GetBlock() != nil {
+			isBlock = true
+		}
+	}
+
+	// We want to fail early if the user is trying to create a RWX on a non-block type device
+	if isMultiNode && !isBlock {
+		return nil, status.Error(codes.InvalidArgument, "multi node access modes are only supported on rbd `block` type volumes")
+	}
+
+	// if it's NOT SINGLE_NODE_WRITER and it's BLOCK we'll set the parameter to ignore the in-use checks
+	rbdVol, err := genVolFromVolumeOptions(ctx, req.GetParameters(), req.GetSecrets(), (isMultiNode && isBlock), false)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	rbdVol.RequestName = req.GetName()
+
+	// Volume Size - Default is 1 GiB
+	volSizeBytes := int64(oneGB)
+	if req.GetCapacityRange() != nil {
+		volSizeBytes = req.GetCapacityRange().GetRequiredBytes()
+	}
+
+	// always round up the request size in bytes to the nearest MiB/GiB
+	rbdVol.VolSize = util.RoundOffBytes(volSizeBytes)
+
+	// start with pool the same as journal pool, in case there is a topology
+	// based split, pool for the image will be updated subsequently
+	rbdVol.JournalPool = rbdVol.Pool
+
+	// store topology information from the request
+	rbdVol.TopologyPools, rbdVol.TopologyRequirement, err = util.GetTopologyFromRequest(req)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	// NOTE: rbdVol does not contain VolID and RbdImageName populated, everything
+	// else is populated post create request parsing
+	return rbdVol, nil
+}
+
+// CreateVolume creates the volume in backend
+func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
+	if err := cs.validateVolumeReq(ctx, req); err != nil {
+		return nil, err
+	}
+
+	// TODO: create/get a connection from the the ConnPool, and do not pass
+	// the credentials to any of the utility functions.
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	rbdVol, err := cs.parseVolCreateRequest(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer rbdVol.Destroy()
+
+	// Existence and conflict checks
+	if acquired := cs.VolumeLocks.TryAcquire(req.GetName()); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), req.GetName())
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetName())
+	}
+	defer cs.VolumeLocks.Release(req.GetName())
+
+	found, err := checkVolExists(ctx, rbdVol, cr)
+	if err != nil {
+		if _, ok := err.(ErrVolNameConflict); ok {
+			return nil, status.Error(codes.AlreadyExists, err.Error())
+		}
+
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	if found {
+		if rbdVol.Encrypted {
+			err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+			if err != nil {
+				klog.Errorf(util.Log(ctx, err.Error()))
+				return nil, err
+			}
+		}
+
+		volumeContext := req.GetParameters()
+		volumeContext["pool"] = rbdVol.Pool
+		volumeContext["journalPool"] = rbdVol.JournalPool
+		volumeContext["imageName"] = rbdVol.RbdImageName
+		volume := &csi.Volume{
+			VolumeId:      rbdVol.VolID,
+			CapacityBytes: rbdVol.VolSize,
+			VolumeContext: volumeContext,
+			ContentSource: req.GetVolumeContentSource(),
+		}
+		if rbdVol.Topology != nil {
+			volume.AccessibleTopology =
+				[]*csi.Topology{
+					{
+						Segments: rbdVol.Topology,
+					},
+				}
+		}
+		return &csi.CreateVolumeResponse{Volume: volume}, nil
+	}
+
+	rbdSnap, err := cs.checkSnapshotSource(ctx, req, cr)
+	if err != nil {
+		return nil, err
+	}
+
+	err = reserveVol(ctx, rbdVol, rbdSnap, cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			errDefer := undoVolReservation(ctx, rbdVol, cr)
+			if errDefer != nil {
+				klog.Warningf(util.Log(ctx, "failed undoing reservation of volume: %s (%s)"), req.GetName(), errDefer)
+			}
+		}
+	}()
+
+	err = createBackingImage(ctx, cr, rbdVol, rbdSnap)
+	if err != nil {
+		return nil, err
+	}
+
+	if rbdVol.Encrypted {
+		err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"),
+				rbdVol.RbdImageName)
+			if deleteErr := deleteImage(ctx, rbdVol, cr); err != nil {
+				klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s with error: %v"),
+					rbdVol.Pool, rbdVol.RbdImageName, deleteErr)
+				return nil, deleteErr
+			}
+			return nil, err
+		}
+	}
+
+	volumeContext := req.GetParameters()
+	volumeContext["pool"] = rbdVol.Pool
+	volumeContext["journalPool"] = rbdVol.JournalPool
+	volumeContext["imageName"] = rbdVol.RbdImageName
+	volume := &csi.Volume{
+		VolumeId:      rbdVol.VolID,
+		CapacityBytes: rbdVol.VolSize,
+		VolumeContext: volumeContext,
+		ContentSource: req.GetVolumeContentSource(),
+	}
+	if rbdVol.Topology != nil {
+		volume.AccessibleTopology =
+			[]*csi.Topology{
+				{
+					Segments: rbdVol.Topology,
+				},
+			}
+	}
+	return &csi.CreateVolumeResponse{Volume: volume}, nil
+}
+
+func createBackingImage(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume, rbdSnap *rbdSnapshot) error {
+	var err error
+
+	if rbdSnap != nil {
+		err = restoreSnapshot(ctx, rbdVol, rbdSnap, cr)
+		if err != nil {
+			return err
+		}
+
+		klog.V(4).Infof(util.Log(ctx, "created volume %s from snapshot %s"), rbdVol.RequestName, rbdSnap.RbdSnapName)
+		return nil
+	}
+
+	err = createImage(ctx, rbdVol, cr)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create volume: %v"), err)
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "created volume %s backed by image %s"), rbdVol.RequestName, rbdVol.RbdImageName)
+
+	return nil
+}
+
+func (cs *ControllerServer) checkSnapshotSource(ctx context.Context, req *csi.CreateVolumeRequest,
+	cr *util.Credentials) (*rbdSnapshot, error) {
+	if req.VolumeContentSource == nil {
+		return nil, nil
+	}
+
+	snapshot := req.VolumeContentSource.GetSnapshot()
+	if snapshot == nil {
+		return nil, status.Error(codes.InvalidArgument, "volume Snapshot cannot be empty")
+	}
+
+	snapshotID := snapshot.GetSnapshotId()
+	if snapshotID == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume Snapshot ID cannot be empty")
+	}
+
+	rbdSnap := &rbdSnapshot{}
+	if err := genSnapFromSnapID(ctx, rbdSnap, snapshotID, cr); err != nil {
+		if _, ok := err.(ErrSnapNotFound); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Errorf(util.Log(ctx, "failed to get backend snapshot for %s: %v"), snapshotID, err)
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		}
+
+		return nil, status.Error(codes.InvalidArgument, "missing requested Snapshot ID")
+	}
+
+	return rbdSnap, nil
+}
+
+// DeleteLegacyVolume deletes a volume provisioned using version 1.0.0 of the plugin
+func (cs *ControllerServer) DeleteLegacyVolume(ctx context.Context, req *csi.DeleteVolumeRequest, cr *util.Credentials) (*csi.DeleteVolumeResponse, error) {
+	volumeID := req.GetVolumeId()
+
+	if cs.MetadataStore == nil {
+		return nil, status.Errorf(codes.InvalidArgument, "missing metadata store configuration to"+
+			" proceed with deleting legacy volume ID (%s)", volumeID)
+	}
+
+	if acquired := cs.VolumeLocks.TryAcquire(volumeID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volumeID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID)
+	}
+	defer cs.VolumeLocks.Release(volumeID)
+
+	rbdVol := &rbdVolume{}
+	defer rbdVol.Destroy()
+	if err := cs.MetadataStore.Get(volumeID, rbdVol); err != nil {
+		if err, ok := err.(*util.CacheEntryNotFound); ok {
+			klog.Warningf(util.Log(ctx, "metadata for legacy volume %s not found, assuming the volume to be already deleted (%v)"), volumeID, err)
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	// Fill up Monitors
+	if err := updateMons(rbdVol, nil, req.GetSecrets()); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	// Update rbdImageName as the VolName when dealing with version 1 volumes
+	rbdVol.RbdImageName = rbdVol.VolName
+
+	klog.V(4).Infof(util.Log(ctx, "deleting legacy volume %s"), rbdVol.VolName)
+	if err := deleteImage(ctx, rbdVol, cr); err != nil {
+		// TODO: can we detect "already deleted" situations here and proceed?
+		klog.Errorf(util.Log(ctx, "failed to delete legacy rbd image: %s/%s with error: %v"), rbdVol.Pool, rbdVol.VolName, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if err := cs.MetadataStore.Delete(volumeID); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	return &csi.DeleteVolumeResponse{}, nil
+}
+
+// DeleteVolume deletes the volume in backend and removes the volume metadata
+// from store
+func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+		klog.Errorf(util.Log(ctx, "invalid delete volume req: %v"), protosanitizer.StripSecrets(req))
+		return nil, err
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	// For now the image get unconditionally deleted, but here retention policy can be checked
+	volumeID := req.GetVolumeId()
+	if volumeID == "" {
+		return nil, status.Error(codes.InvalidArgument, "empty volume ID in request")
+	}
+
+	if acquired := cs.VolumeLocks.TryAcquire(volumeID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volumeID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID)
+	}
+	defer cs.VolumeLocks.Release(volumeID)
+
+	rbdVol := &rbdVolume{}
+	defer rbdVol.Destroy()
+	if err = genVolFromVolID(ctx, rbdVol, volumeID, cr, req.GetSecrets()); err != nil {
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Warningf(util.Log(ctx, "failed to get backend volume for %s: %v"), volumeID, err)
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		// If error is ErrInvalidVolID it could be a version 1.0.0 or lower volume, attempt
+		// to process it as such
+		if _, ok := err.(ErrInvalidVolID); ok {
+			if isLegacyVolumeID(volumeID) {
+				klog.V(2).Infof(util.Log(ctx, "attempting deletion of potential legacy volume (%s)"), volumeID)
+				return cs.DeleteLegacyVolume(ctx, req, cr)
+			}
+
+			// Consider unknown volumeID as a successfully deleted volume
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		// if error is ErrKeyNotFound, then a previous attempt at deletion was complete
+		// or partially complete (image and imageOMap are garbage collected already), hence return
+		// success as deletion is complete
+		if _, ok := err.(util.ErrKeyNotFound); ok {
+			klog.Warningf(util.Log(ctx, "Failed to volume options for %s: %v"), volumeID, err)
+			return &csi.DeleteVolumeResponse{}, nil
+		}
+
+		// All errors other than ErrImageNotFound should return an error back to the caller
+		if _, ok := err.(ErrImageNotFound); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// If error is ErrImageNotFound then we failed to find the image, but found the imageOMap
+		// to lead us to the image, hence the imageOMap needs to be garbage collected, by calling
+		// unreserve for the same
+		if acquired := cs.VolumeLocks.TryAcquire(rbdVol.RequestName); !acquired {
+			klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), rbdVol.RequestName)
+			return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName)
+		}
+		defer cs.VolumeLocks.Release(rbdVol.RequestName)
+
+		if err = undoVolReservation(ctx, rbdVol, cr); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		return &csi.DeleteVolumeResponse{}, nil
+	}
+
+	// lock out parallel create requests against the same volume name as we
+	// cleanup the image and associated omaps for the same
+	if acquired := cs.VolumeLocks.TryAcquire(rbdVol.RequestName); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), rbdVol.RequestName)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName)
+	}
+	defer cs.VolumeLocks.Release(rbdVol.RequestName)
+
+	// Deleting rbd image
+	klog.V(4).Infof(util.Log(ctx, "deleting image %s"), rbdVol.RbdImageName)
+	if err = deleteImage(ctx, rbdVol, cr); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s with error: %v"),
+			rbdVol.Pool, rbdVol.RbdImageName, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if err = undoVolReservation(ctx, rbdVol, cr); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)"),
+			rbdVol.RequestName, rbdVol.RbdImageName, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if rbdVol.Encrypted {
+		if err = rbdVol.KMS.DeletePassphrase(rbdVol.VolID); err != nil {
+			klog.Warningf(util.Log(ctx, "failed to clean the passphrase for volume %s: %s"), rbdVol.VolID, err)
+		}
+	}
+
+	return &csi.DeleteVolumeResponse{}, nil
+}
+
+// ValidateVolumeCapabilities checks whether the volume capabilities requested
+// are supported.
+func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
+	if req.GetVolumeId() == "" {
+		return nil, status.Error(codes.InvalidArgument, "empty volume ID in request")
+	}
+
+	if len(req.VolumeCapabilities) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "empty volume capabilities in request")
+	}
+
+	for _, cap := range req.VolumeCapabilities {
+		if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+			return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
+		}
+	}
+	return &csi.ValidateVolumeCapabilitiesResponse{
+		Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
+			VolumeCapabilities: req.VolumeCapabilities,
+		},
+	}, nil
+}
+
+// CreateSnapshot creates the snapshot in backend and stores metadata
+// in store
+// nolint: gocyclo
+func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
+	if err := cs.validateSnapshotReq(ctx, req); err != nil {
+		return nil, err
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	// Fetch source volume information
+	rbdVol := new(rbdVolume)
+	err = genVolFromVolID(ctx, rbdVol, req.GetSourceVolumeId(), cr, req.GetSecrets())
+	if err != nil {
+		if _, ok := err.(ErrImageNotFound); ok {
+			return nil, status.Errorf(codes.NotFound, "source Volume ID %s not found", req.GetSourceVolumeId())
+		}
+
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Errorf(util.Log(ctx, "failed to get backend volume for %s: %v"), req.GetSourceVolumeId(), err)
+			return nil, status.Errorf(codes.Internal, err.Error())
+		}
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+
+	// TODO: re-encrypt snapshot with a new passphrase
+	if rbdVol.Encrypted {
+		return nil, status.Errorf(codes.Unimplemented, "source Volume %s is encrypted, "+
+			"snapshotting is not supported currently", rbdVol.VolID)
+	}
+
+	// Check if source volume was created with required image features for snaps
+	if !hasSnapshotFeature(rbdVol.ImageFeatures) {
+		return nil, status.Errorf(codes.InvalidArgument, "volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId())
+	}
+
+	// Create snap volume
+	rbdSnap := genSnapFromOptions(ctx, rbdVol, req.GetParameters())
+	rbdSnap.RbdImageName = rbdVol.RbdImageName
+	rbdSnap.SizeBytes = rbdVol.VolSize
+	rbdSnap.SourceVolumeID = req.GetSourceVolumeId()
+	rbdSnap.RequestName = req.GetName()
+
+	if acquired := cs.SnapshotLocks.TryAcquire(req.GetName()); !acquired {
+		klog.Errorf(util.Log(ctx, util.SnapshotOperationAlreadyExistsFmt), req.GetName())
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetName())
+	}
+	defer cs.SnapshotLocks.Release(req.GetName())
+
+	// Need to check for already existing snapshot name, and if found
+	// check for the requested source volume id and already allocated source volume id
+	found, err := checkSnapExists(ctx, rbdSnap, cr)
+	if err != nil {
+		if _, ok := err.(util.ErrSnapNameConflict); ok {
+			return nil, status.Error(codes.AlreadyExists, err.Error())
+		}
+
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+	if found {
+		return &csi.CreateSnapshotResponse{
+			Snapshot: &csi.Snapshot{
+				SizeBytes:      rbdSnap.SizeBytes,
+				SnapshotId:     rbdSnap.SnapID,
+				SourceVolumeId: rbdSnap.SourceVolumeID,
+				CreationTime:   rbdSnap.CreatedAt,
+				ReadyToUse:     true,
+			},
+		}, nil
+	}
+
+	err = reserveSnap(ctx, rbdSnap, cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			errDefer := undoSnapReservation(ctx, rbdSnap, cr)
+			if errDefer != nil {
+				klog.Warningf(util.Log(ctx, "failed undoing reservation of snapshot: %s %v"), req.GetName(), errDefer)
+			}
+		}
+	}()
+
+	err = cs.doSnapshot(ctx, rbdSnap, cr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &csi.CreateSnapshotResponse{
+		Snapshot: &csi.Snapshot{
+			SizeBytes:      rbdSnap.SizeBytes,
+			SnapshotId:     rbdSnap.SnapID,
+			SourceVolumeId: req.GetSourceVolumeId(),
+			CreationTime:   rbdSnap.CreatedAt,
+			ReadyToUse:     true,
+		},
+	}, nil
+}
+
+func (cs *ControllerServer) validateSnapshotReq(ctx context.Context, req *csi.CreateSnapshotRequest) error {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
+		klog.Errorf(util.Log(ctx, "invalid create snapshot req: %v"), protosanitizer.StripSecrets(req))
+		return err
+	}
+
+	// Check sanity of request Snapshot Name, Source Volume Id
+	if req.Name == "" {
+		return status.Error(codes.InvalidArgument, "snapshot Name cannot be empty")
+	}
+	if req.SourceVolumeId == "" {
+		return status.Error(codes.InvalidArgument, "source Volume ID cannot be empty")
+	}
+
+	options := req.GetParameters()
+	if value, ok := options["snapshotNamePrefix"]; ok && value == "" {
+		return status.Error(codes.InvalidArgument, "empty snapshot name prefix to provision snapshot from")
+	}
+
+	return nil
+}
+
+func (cs *ControllerServer) doSnapshot(ctx context.Context, rbdSnap *rbdSnapshot, cr *util.Credentials) (err error) {
+	err = createSnapshot(ctx, rbdSnap, cr)
+	// If snap creation fails, even due to snapname already used, fail, next attempt will get a new
+	// uuid for use as the snap name
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to create snapshot: %v"), err)
+		return status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			errDefer := deleteSnapshot(ctx, rbdSnap, cr)
+			if errDefer != nil {
+				klog.Errorf(util.Log(ctx, "failed to delete snapshot: %v"), errDefer)
+				err = fmt.Errorf("snapshot created but failed to delete snapshot due to"+
+					" other failures: %v", err)
+			}
+			err = status.Error(codes.Internal, err.Error())
+		}
+	}()
+	err = protectSnapshot(ctx, rbdSnap, cr)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to protect snapshot: %v"), err)
+		return status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			errDefer := unprotectSnapshot(ctx, rbdSnap, cr)
+			if errDefer != nil {
+				klog.Errorf(util.Log(ctx, "failed to unprotect snapshot: %v"), errDefer)
+				err = fmt.Errorf("snapshot created but failed to unprotect snapshot due to"+
+					" other failures: %v", err)
+			}
+			err = status.Error(codes.Internal, err.Error())
+		}
+	}()
+
+	err = getSnapshotMetadata(ctx, rbdSnap, cr)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to fetch snapshot metadata: %v"), err)
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	return nil
+}
+
+// DeleteSnapshot deletes the snapshot in backend and removes the
+// snapshot metadata from store
+func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
+		klog.Errorf(util.Log(ctx, "invalid delete snapshot req: %v"), protosanitizer.StripSecrets(req))
+		return nil, err
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	snapshotID := req.GetSnapshotId()
+	if snapshotID == "" {
+		return nil, status.Error(codes.InvalidArgument, "snapshot ID cannot be empty")
+	}
+
+	if acquired := cs.SnapshotLocks.TryAcquire(snapshotID); !acquired {
+		klog.Errorf(util.Log(ctx, util.SnapshotOperationAlreadyExistsFmt), snapshotID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, snapshotID)
+	}
+	defer cs.SnapshotLocks.Release(snapshotID)
+
+	rbdSnap := &rbdSnapshot{}
+	if err = genSnapFromSnapID(ctx, rbdSnap, snapshotID, cr); err != nil {
+		// if error is ErrPoolNotFound, the pool is already deleted we dont
+		// need to worry about deleting snapshot or omap data, return success
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Warningf(util.Log(ctx, "failed to get backend snapshot for %s: %v"), snapshotID, err)
+			return &csi.DeleteSnapshotResponse{}, nil
+		}
+
+		// if error is ErrKeyNotFound, then a previous attempt at deletion was complete
+		// or partially complete (snap and snapOMap are garbage collected already), hence return
+		// success as deletion is complete
+		if _, ok := err.(util.ErrKeyNotFound); ok {
+			return &csi.DeleteSnapshotResponse{}, nil
+		}
+
+		// All errors other than ErrSnapNotFound should return an error back to the caller
+		if _, ok := err.(ErrSnapNotFound); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// Consider missing snap as already deleted, and proceed to remove the omap values,
+		// safeguarding against parallel create or delete requests against the
+		// same name.
+		if acquired := cs.SnapshotLocks.TryAcquire(rbdSnap.RequestName); !acquired {
+			klog.Errorf(util.Log(ctx, util.SnapshotOperationAlreadyExistsFmt), rbdSnap.RequestName)
+			return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdSnap.RequestName)
+		}
+		defer cs.SnapshotLocks.Release(rbdSnap.RequestName)
+
+		if err = undoSnapReservation(ctx, rbdSnap, cr); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		return &csi.DeleteSnapshotResponse{}, nil
+	}
+
+	// safeguard against parallel create or delete requests against the same
+	// name
+	if acquired := cs.SnapshotLocks.TryAcquire(rbdSnap.RequestName); !acquired {
+		klog.Errorf(util.Log(ctx, util.SnapshotOperationAlreadyExistsFmt), rbdSnap.RequestName)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdSnap.RequestName)
+	}
+	defer cs.SnapshotLocks.Release(rbdSnap.RequestName)
+
+	// Unprotect snapshot
+	err = unprotectSnapshot(ctx, rbdSnap, cr)
+	if err != nil {
+		return nil, status.Errorf(codes.FailedPrecondition,
+			"failed to unprotect snapshot: %s/%s with error: %v",
+			rbdSnap.Pool, rbdSnap.RbdSnapName, err)
+	}
+
+	// Deleting snapshot
+	klog.V(4).Infof(util.Log(ctx, "deleting Snaphot %s"), rbdSnap.RbdSnapName)
+	if err := deleteSnapshot(ctx, rbdSnap, cr); err != nil {
+		return nil, status.Errorf(codes.FailedPrecondition,
+			"failed to delete snapshot: %s/%s with error: %v",
+			rbdSnap.Pool, rbdSnap.RbdSnapName, err)
+	}
+
+	return &csi.DeleteSnapshotResponse{}, nil
+}
+
+// ControllerExpandVolume expand RBD Volumes on demand based on resizer request
+func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil {
+		klog.Errorf(util.Log(ctx, "invalid expand volume req: %v"), protosanitizer.StripSecrets(req))
+		return nil, err
+	}
+
+	volID := req.GetVolumeId()
+	if volID == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume ID cannot be empty")
+	}
+
+	capRange := req.GetCapacityRange()
+	if capRange == nil {
+		return nil, status.Error(codes.InvalidArgument, "capacityRange cannot be empty")
+	}
+
+	// lock out parallel requests against the same volume ID
+	if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer cs.VolumeLocks.Release(volID)
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	rbdVol := &rbdVolume{}
+	defer rbdVol.Destroy()
+	err = genVolFromVolID(ctx, rbdVol, volID, cr, req.GetSecrets())
+	if err != nil {
+		if _, ok := err.(ErrImageNotFound); ok {
+			return nil, status.Errorf(codes.NotFound, "volume ID %s not found", volID)
+		}
+
+		if _, ok := err.(util.ErrPoolNotFound); ok {
+			klog.Errorf(util.Log(ctx, "failed to get backend volume for %s: %v"), volID, err)
+			return nil, status.Errorf(codes.InvalidArgument, err.Error())
+		}
+
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+
+	if rbdVol.Encrypted {
+		return nil, status.Errorf(codes.InvalidArgument, "encrypted volumes do not support resize (%s/%s)",
+			rbdVol.Pool, rbdVol.RbdImageName)
+	}
+
+	// always round up the request size in bytes to the nearest MiB/GiB
+	volSize := util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes())
+
+	// resize volume if required
+	nodeExpansion := false
+	if rbdVol.VolSize < volSize {
+		klog.V(4).Infof(util.Log(ctx, "rbd volume %s/%s size is %v,resizing to %v"), rbdVol.Pool, rbdVol.RbdImageName, rbdVol.VolSize, volSize)
+		rbdVol.VolSize = volSize
+		nodeExpansion = true
+		err = resizeRBDImage(rbdVol, cr)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to resize rbd image: %s/%s with error: %v"), rbdVol.Pool, rbdVol.RbdImageName, err)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	return &csi.ControllerExpandVolumeResponse{
+		CapacityBytes:         rbdVol.VolSize,
+		NodeExpansionRequired: nodeExpansion,
+	}, nil
+}

internal/rbd/driver.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"k8s.io/klog"
+	"k8s.io/utils/mount"
+)
+
+const (
+	// volIDVersion is the version number of volume ID encoding scheme
+	volIDVersion uint16 = 1
+
+	// csiConfigFile is the location of the CSI config file
+	csiConfigFile = "/etc/ceph-csi-config/config.json"
+)
+
+// Driver contains the default identity,node and controller struct
+type Driver struct {
+	cd *csicommon.CSIDriver
+
+	ids *IdentityServer
+	ns  *NodeServer
+	cs  *ControllerServer
+}
+
+var (
+
+	// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
+	// ceph clusters across CSI instances, to differentiate omap names per CSI instance
+	CSIInstanceID = "default"
+
+	// volJournal and snapJournal are used to maintain RADOS based journals for CO generated
+	// VolumeName to backing RBD images
+	volJournal  *util.CSIJournal
+	snapJournal *util.CSIJournal
+)
+
+// NewDriver returns new rbd driver
+func NewDriver() *Driver {
+	return &Driver{}
+}
+
+// NewIdentityServer initialize a identity server for rbd CSI driver
+func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer {
+	return &IdentityServer{
+		DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
+	}
+}
+
+// NewControllerServer initialize a controller server for rbd CSI driver
+func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer {
+	return &ControllerServer{
+		DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
+		MetadataStore:           cachePersister,
+		VolumeLocks:             util.NewVolumeLocks(),
+		SnapshotLocks:           util.NewVolumeLocks(),
+	}
+}
+
+// NewNodeServer initialize a node server for rbd CSI driver.
+func NewNodeServer(d *csicommon.CSIDriver, t string, topology map[string]string) (*NodeServer, error) {
+	mounter := mount.New("")
+	return &NodeServer{
+		DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t, topology),
+		mounter:           mounter,
+		VolumeLocks:       util.NewVolumeLocks(),
+	}, nil
+}
+
+// Run start a non-blocking grpc controller,node and identityserver for
+// rbd CSI driver which can serve multiple parallel requests
+func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
+	var err error
+	var topology map[string]string
+
+	// Create ceph.conf for use with CLI commands
+	if err = util.WriteCephConfig(); err != nil {
+		klog.Fatalf("failed to write ceph configuration file (%v)", err)
+	}
+
+	// Use passed in instance ID, if provided for omap suffix naming
+	if conf.InstanceID != "" {
+		CSIInstanceID = conf.InstanceID
+	}
+
+	// Get an instance of the volume and snapshot journal keys
+	volJournal = util.NewCSIVolumeJournal()
+	snapJournal = util.NewCSISnapshotJournal()
+
+	// Update keys with CSI instance suffix
+	volJournal.SetCSIDirectorySuffix(CSIInstanceID)
+	snapJournal.SetCSIDirectorySuffix(CSIInstanceID)
+
+	// Initialize default library driver
+	r.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID)
+	if r.cd == nil {
+		klog.Fatalln("Failed to initialize CSI Driver.")
+	}
+	if conf.IsControllerServer || !conf.IsNodeServer {
+		r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
+			csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
+			csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+			csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
+			csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
+		})
+		// We only support the multi-writer option when using block, but it's a supported capability for the plugin in general
+		// In addition, we want to add the remaining modes like MULTI_NODE_READER_ONLY,
+		// MULTI_NODE_SINGLE_WRITER etc, but need to do some verification of RO modes first
+		// will work those as follow up features
+		r.cd.AddVolumeCapabilityAccessModes(
+			[]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+				csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER})
+	}
+
+	// Create GRPC servers
+	r.ids = NewIdentityServer(r.cd)
+
+	if conf.IsNodeServer {
+		topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName)
+		if err != nil {
+			klog.Fatalln(err)
+		}
+		r.ns, err = NewNodeServer(r.cd, conf.Vtype, topology)
+		if err != nil {
+			klog.Fatalf("failed to start node server, err %v\n", err)
+		}
+	}
+
+	if conf.IsControllerServer {
+		r.cs = NewControllerServer(r.cd, cachePersister)
+	}
+	if !conf.IsControllerServer && !conf.IsNodeServer {
+		topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName)
+		if err != nil {
+			klog.Fatalln(err)
+		}
+		r.ns, err = NewNodeServer(r.cd, conf.Vtype, topology)
+		if err != nil {
+			klog.Fatalf("failed to start node server, err %v\n", err)
+		}
+		r.cs = NewControllerServer(r.cd, cachePersister)
+	}
+
+	s := csicommon.NewNonBlockingGRPCServer()
+	s.Start(conf.Endpoint, conf.HistogramOption, r.ids, r.cs, r.ns, conf.EnableGRPCMetrics)
+	if conf.EnableGRPCMetrics {
+		klog.Warning("EnableGRPCMetrics is deprecated")
+		go util.StartMetricsServer(conf)
+	}
+	s.Wait()
+}

internal/rbd/errors.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+// ErrImageNotFound is returned when image name is not found in the cluster on the given pool
+type ErrImageNotFound struct {
+	imageName string
+	err       error
+}
+
+func (e ErrImageNotFound) Error() string {
+	return e.err.Error()
+}
+
+// ErrSnapNotFound is returned when snap name passed is not found in the list of snapshots for the
+// given image
+type ErrSnapNotFound struct {
+	snapName string
+	err      error
+}
+
+func (e ErrSnapNotFound) Error() string {
+	return e.err.Error()
+}
+
+// ErrVolNameConflict is generated when a requested CSI volume name already exists on RBD but with
+// different properties, and hence is in conflict with the passed in CSI volume name
+type ErrVolNameConflict struct {
+	requestName string
+	err         error
+}
+
+func (e ErrVolNameConflict) Error() string {
+	return e.err.Error()
+}
+
+// ErrInvalidVolID is returned when a CSI passed VolumeID does not conform to any known volume ID
+// formats
+type ErrInvalidVolID struct {
+	err error
+}
+
+func (e ErrInvalidVolID) Error() string {
+	return e.err.Error()
+}
+
+// ErrMissingStash is returned when the image metadata stash file is not found
+type ErrMissingStash struct {
+	err error
+}
+
+func (e ErrMissingStash) Error() string {
+	return e.err.Error()
+}

internal/rbd/identityserver.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+// IdentityServer struct of rbd CSI driver with supported methods of CSI
+// identity server spec.
+type IdentityServer struct {
+	*csicommon.DefaultIdentityServer
+}
+
+// GetPluginCapabilities returns available capabilities of the rbd driver
+func (is *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
+	return &csi.GetPluginCapabilitiesResponse{
+		Capabilities: []*csi.PluginCapability{
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
+					},
+				},
+			},
+			{
+				Type: &csi.PluginCapability_VolumeExpansion_{
+					VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
+						Type: csi.PluginCapability_VolumeExpansion_ONLINE,
+					},
+				},
+			},
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS,
+					},
+				},
+			},
+		},
+	}, nil
+}

internal/rbd/nodeserver.go

@@ -0,0 +1,802 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/util/resizefs"
+	utilexec "k8s.io/utils/exec"
+	"k8s.io/utils/mount"
+)
+
+// NodeServer struct of ceph rbd driver with supported methods of CSI
+// node server spec
+type NodeServer struct {
+	*csicommon.DefaultNodeServer
+	mounter mount.Interface
+	// A map storing all volumes with ongoing operations so that additional operations
+	// for that same volume (as defined by VolumeID) return an Aborted error
+	VolumeLocks *util.VolumeLocks
+}
+
+// stageTransaction struct represents the state a transaction was when it either completed
+// or failed
+// this transaction state can be used to rollback the transaction
+type stageTransaction struct {
+	// isStagePathCreated represents whether the mount path to stage the volume on was created or not
+	isStagePathCreated bool
+	// isMounted represents if the volume was mounted or not
+	isMounted bool
+	// isEncrypted represents if the volume was encrypted or not
+	isEncrypted bool
+}
+
+// NodeStageVolume mounts the volume to a staging path on the node.
+// Implementation notes:
+// - stagingTargetPath is the directory passed in the request where the volume needs to be staged
+//   - We stage the volume into a directory, named after the VolumeID inside stagingTargetPath if
+//    it is a file system
+//   - We stage the volume into a file, named after the VolumeID inside stagingTargetPath if it is
+//    a block volume
+// - Order of operation execution: (useful for defer stacking and when Unstaging to ensure steps
+//	are done in reverse, this is done in undoStagingTransaction)
+//   - Stash image metadata under staging path
+//   - Map the image (creates a device)
+//   - Create the staging file/directory under staging path
+//   - Stage the device (mount the device mapped for image)
+// nolint: gocyclo
+func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+	if err := util.ValidateNodeStageVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	isBlock := req.GetVolumeCapability().GetBlock() != nil
+	disableInUseChecks := false
+	// MULTI_NODE_MULTI_WRITER is supported by default for Block access type volumes
+	if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER {
+		if !isBlock {
+			klog.Warningf(util.Log(ctx, "MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`, invalid AccessMode for volume: %v"), req.GetVolumeId())
+			return nil, status.Error(codes.InvalidArgument, "rbd: RWX access mode request is only valid for volumes with access type `block`")
+		}
+
+		disableInUseChecks = true
+	}
+
+	volID := req.GetVolumeId()
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	stagingParentPath := req.GetStagingTargetPath()
+	stagingTargetPath := stagingParentPath + "/" + volID
+
+	// check is it a static volume
+	staticVol := false
+	val, ok := req.GetVolumeContext()["staticVolume"]
+	if ok {
+		if staticVol, err = strconv.ParseBool(val); err != nil {
+			return nil, status.Error(codes.InvalidArgument, err.Error())
+		}
+	}
+
+	var isNotMnt bool
+	// check if stagingPath is already mounted
+	isNotMnt, err = mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if !isNotMnt {
+		klog.V(4).Infof(util.Log(ctx, "rbd: volume %s is already mounted to %s, skipping"), volID, stagingTargetPath)
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
+	isLegacyVolume := isLegacyVolumeID(volID)
+	volOptions, err := genVolFromVolumeOptions(ctx, req.GetVolumeContext(), req.GetSecrets(), disableInUseChecks, isLegacyVolume)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	// get rbd image name from the volume journal
+	// for static volumes, the image name is actually the volume ID itself
+	// for legacy volumes (v1.0.0), the image name can be found in the staging path
+	switch {
+	case staticVol:
+		volOptions.RbdImageName = volID
+	case isLegacyVolume:
+		volOptions.RbdImageName, err = getLegacyVolumeName(stagingTargetPath)
+		if err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	default:
+		var vi util.CSIIdentifier
+		var imageAttributes *util.ImageAttributes
+		err = vi.DecomposeCSIID(volID)
+		if err != nil {
+			err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		imageAttributes, err = volJournal.GetImageAttributes(ctx, volOptions.Monitors, cr,
+			volOptions.Pool, vi.ObjectUUID, false)
+		if err != nil {
+			err = fmt.Errorf("error fetching image attributes for volume ID (%s) (%s)", err, volID)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		volOptions.RbdImageName = imageAttributes.ImageName
+	}
+
+	volOptions.VolID = volID
+	transaction := stageTransaction{}
+	devicePath := ""
+
+	// Stash image details prior to mapping the image (useful during Unstage as it has no
+	// voloptions passed to the RPC as per the CSI spec)
+	err = stashRBDImageMetadata(volOptions, stagingParentPath)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer func() {
+		if err != nil {
+			ns.undoStagingTransaction(ctx, req, devicePath, transaction)
+		}
+	}()
+
+	// perform the actual staging and if this fails, have undoStagingTransaction
+	// cleans up for us
+	transaction, err = ns.stageTransaction(ctx, req, volOptions, staticVol)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: successfully mounted volume %s to stagingTargetPath %s"), req.GetVolumeId(), stagingTargetPath)
+
+	return &csi.NodeStageVolumeResponse{}, nil
+}
+
+func (ns *NodeServer) stageTransaction(ctx context.Context, req *csi.NodeStageVolumeRequest, volOptions *rbdVolume, staticVol bool) (stageTransaction, error) {
+	transaction := stageTransaction{}
+
+	var err error
+
+	var cr *util.Credentials
+	cr, err = util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return transaction, err
+	}
+	defer cr.DeleteCredentials()
+
+	// Mapping RBD image
+	var devicePath string
+	devicePath, err = attachRBDImage(ctx, volOptions, cr)
+	if err != nil {
+		return transaction, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"),
+		req.GetVolumeId(), volOptions.Pool, devicePath)
+
+	if volOptions.Encrypted {
+		devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr)
+		if err != nil {
+			return transaction, err
+		}
+		transaction.isEncrypted = true
+	}
+
+	stagingTargetPath := getStagingTargetPath(req)
+
+	isBlock := req.GetVolumeCapability().GetBlock() != nil
+	err = ns.createStageMountPoint(ctx, stagingTargetPath, isBlock)
+	if err != nil {
+		return transaction, err
+	}
+
+	transaction.isStagePathCreated = true
+
+	// nodeStage Path
+	err = ns.mountVolumeToStagePath(ctx, req, staticVol, stagingTargetPath, devicePath)
+	if err != nil {
+		return transaction, err
+	}
+	transaction.isMounted = true
+
+	// #nosec - allow anyone to write inside the target path
+	err = os.Chmod(stagingTargetPath, 0777)
+
+	return transaction, err
+}
+
+func (ns *NodeServer) undoStagingTransaction(ctx context.Context, req *csi.NodeStageVolumeRequest, devicePath string, transaction stageTransaction) {
+	var err error
+
+	stagingTargetPath := getStagingTargetPath(req)
+	if transaction.isMounted {
+		err = ns.mounter.Unmount(stagingTargetPath)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to unmount stagingtargetPath: %s with error: %v"), stagingTargetPath, err)
+			return
+		}
+	}
+
+	// remove the file/directory created on staging path
+	if transaction.isStagePathCreated {
+		err = os.Remove(stagingTargetPath)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to remove stagingtargetPath: %s with error: %v"), stagingTargetPath, err)
+			// continue on failure to unmap the image, as leaving stale images causes more issues than a stale file/directory
+		}
+	}
+
+	volID := req.GetVolumeId()
+
+	// Unmapping rbd device
+	if devicePath != "" {
+		err = detachRBDDevice(ctx, devicePath, volID, transaction.isEncrypted)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to unmap rbd device: %s for volume %s with error: %v"), devicePath, volID, err)
+			// continue on failure to delete the stash file, as kubernetes will fail to delete the staging path otherwise
+		}
+	}
+
+	// Cleanup the stashed image metadata
+	if err = cleanupRBDImageMetadataStash(req.GetStagingTargetPath()); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to cleanup image metadata stash (%v)"), err)
+		return
+	}
+}
+
+func (ns *NodeServer) createStageMountPoint(ctx context.Context, mountPath string, isBlock bool) error {
+	if isBlock {
+		pathFile, err := os.OpenFile(mountPath, os.O_CREATE|os.O_RDWR, 0600)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to create mountPath:%s with error: %v"), mountPath, err)
+			return status.Error(codes.Internal, err.Error())
+		}
+		if err = pathFile.Close(); err != nil {
+			klog.Errorf(util.Log(ctx, "failed to close mountPath:%s with error: %v"), mountPath, err)
+			return status.Error(codes.Internal, err.Error())
+		}
+
+		return nil
+	}
+
+	err := os.Mkdir(mountPath, 0750)
+	if err != nil {
+		if !os.IsExist(err) {
+			klog.Errorf(util.Log(ctx, "failed to create mountPath:%s with error: %v"), mountPath, err)
+			return status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	return nil
+}
+
+// NodePublishVolume mounts the volume mounted to the device path to the target
+// path
+func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
+	err := util.ValidateNodePublishVolumeRequest(req)
+	if err != nil {
+		return nil, err
+	}
+	targetPath := req.GetTargetPath()
+	isBlock := req.GetVolumeCapability().GetBlock() != nil
+	stagingPath := req.GetStagingTargetPath()
+	volID := req.GetVolumeId()
+	stagingPath += "/" + volID
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	// Check if that target path exists properly
+	notMnt, err := ns.createTargetMountPath(ctx, targetPath, isBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	if !notMnt {
+		return &csi.NodePublishVolumeResponse{}, nil
+	}
+
+	// Publish Path
+	err = ns.mountVolume(ctx, stagingPath, req)
+	if err != nil {
+		return nil, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: successfully mounted stagingPath %s to targetPath %s"), stagingPath, targetPath)
+	return &csi.NodePublishVolumeResponse{}, nil
+}
+
+func getLegacyVolumeName(mountPath string) (string, error) {
+	var volName string
+
+	if strings.HasSuffix(mountPath, "/globalmount") {
+		s := strings.Split(strings.TrimSuffix(mountPath, "/globalmount"), "/")
+		volName = s[len(s)-1]
+		return volName, nil
+	}
+
+	if strings.HasSuffix(mountPath, "/mount") {
+		s := strings.Split(strings.TrimSuffix(mountPath, "/mount"), "/")
+		volName = s[len(s)-1]
+		return volName, nil
+	}
+
+	// get volume name for block volume
+	s := strings.Split(mountPath, "/")
+	if len(s) == 0 {
+		return "", fmt.Errorf("rbd: malformed value of stage target path: %s", mountPath)
+	}
+	volName = s[len(s)-1]
+	return volName, nil
+}
+
+func (ns *NodeServer) mountVolumeToStagePath(ctx context.Context, req *csi.NodeStageVolumeRequest, staticVol bool, stagingPath, devicePath string) error {
+	fsType := req.GetVolumeCapability().GetMount().GetFsType()
+	diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+	// rbd images are thin-provisioned and return zeros for unwritten areas.  A freshly created
+	// image will not benefit from discard and we also want to avoid as much unnecessary zeroing
+	// as possible.  Open-code mkfs here because FormatAndMount() doesn't accept custom mkfs
+	// options.
+	//
+	// Note that "freshly" is very important here.  While discard is more of a nice to have,
+	// lazy_journal_init=1 is plain unsafe if the image has been written to before and hasn't
+	// been zeroed afterwards (unlike the name suggests, it leaves the journal completely
+	// uninitialized and carries a risk until the journal is overwritten and wraps around for
+	// the first time).
+	existingFormat, err := diskMounter.GetDiskFormat(devicePath)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get disk format for path %s, error: %v"), devicePath, err)
+		return err
+	}
+
+	if existingFormat == "" && !staticVol {
+		args := []string{}
+		if fsType == "ext4" {
+			args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
+		} else if fsType == "xfs" {
+			args = []string{"-K", devicePath}
+		}
+		if len(args) > 0 {
+			cmdOut, cmdErr := diskMounter.Exec.Command("mkfs."+fsType, args...).CombinedOutput()
+			if cmdErr != nil {
+				klog.Errorf(util.Log(ctx, "failed to run mkfs error: %v, output: %v"), cmdErr, cmdOut)
+				return cmdErr
+			}
+		}
+	}
+
+	opt := []string{"_netdev"}
+	opt = csicommon.ConstructMountOptions(opt, req.GetVolumeCapability())
+	isBlock := req.GetVolumeCapability().GetBlock() != nil
+
+	if isBlock {
+		opt = append(opt, "bind")
+		err = diskMounter.Mount(devicePath, stagingPath, fsType, opt)
+	} else {
+		err = diskMounter.FormatAndMount(devicePath, stagingPath, fsType, opt)
+	}
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to mount device path (%s) to staging path (%s) for volume (%s) error %s"), devicePath, stagingPath, req.GetVolumeId(), err)
+	}
+	return err
+}
+
+func (ns *NodeServer) mountVolume(ctx context.Context, stagingPath string, req *csi.NodePublishVolumeRequest) error {
+	// Publish Path
+	fsType := req.GetVolumeCapability().GetMount().GetFsType()
+	readOnly := req.GetReadonly()
+	mountOptions := []string{"bind", "_netdev"}
+	isBlock := req.GetVolumeCapability().GetBlock() != nil
+	targetPath := req.GetTargetPath()
+
+	mountOptions = csicommon.ConstructMountOptions(mountOptions, req.GetVolumeCapability())
+
+	klog.V(4).Infof(util.Log(ctx, "target %v\nisBlock %v\nfstype %v\nstagingPath %v\nreadonly %v\nmountflags %v\n"),
+		targetPath, isBlock, fsType, stagingPath, readOnly, mountOptions)
+
+	if readOnly {
+		mountOptions = append(mountOptions, "ro")
+	}
+	if err := util.Mount(stagingPath, targetPath, fsType, mountOptions); err != nil {
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	return nil
+}
+
+func (ns *NodeServer) createTargetMountPath(ctx context.Context, mountPath string, isBlock bool) (bool, error) {
+	// Check if that mount path exists properly
+	notMnt, err := mount.IsNotMountPoint(ns.mounter, mountPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if isBlock {
+				// #nosec
+				pathFile, e := os.OpenFile(mountPath, os.O_CREATE|os.O_RDWR, 0750)
+				if e != nil {
+					klog.V(4).Infof(util.Log(ctx, "Failed to create mountPath:%s with error: %v"), mountPath, err)
+					return notMnt, status.Error(codes.Internal, e.Error())
+				}
+				if err = pathFile.Close(); err != nil {
+					klog.V(4).Infof(util.Log(ctx, "Failed to close mountPath:%s with error: %v"), mountPath, err)
+					return notMnt, status.Error(codes.Internal, err.Error())
+				}
+			} else {
+				// Create a directory
+				if err = util.CreateMountPoint(mountPath); err != nil {
+					return notMnt, status.Error(codes.Internal, err.Error())
+				}
+			}
+			notMnt = true
+		} else {
+			return false, status.Error(codes.Internal, err.Error())
+		}
+	}
+	return notMnt, err
+}
+
+// NodeUnpublishVolume unmounts the volume from the target path
+func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
+	err := util.ValidateNodeUnpublishVolumeRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	targetPath := req.GetTargetPath()
+	volID := req.GetVolumeId()
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	notMnt, err := mount.IsNotMountPoint(ns.mounter, targetPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// targetPath has already been deleted
+			klog.V(4).Infof(util.Log(ctx, "targetPath: %s has already been deleted"), targetPath)
+			return &csi.NodeUnpublishVolumeResponse{}, nil
+		}
+		return nil, status.Error(codes.NotFound, err.Error())
+	}
+	if notMnt {
+		if err = os.RemoveAll(targetPath); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		return &csi.NodeUnpublishVolumeResponse{}, nil
+	}
+
+	if err = ns.mounter.Unmount(targetPath); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if err = os.RemoveAll(targetPath); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: successfully unbound volume %s from %s"), req.GetVolumeId(), targetPath)
+
+	return &csi.NodeUnpublishVolumeResponse{}, nil
+}
+
+// getStagingTargetPath concats either NodeStageVolumeRequest's or
+// NodeUnstageVolumeRequest's target path with the volumeID
+func getStagingTargetPath(req interface{}) string {
+	switch vr := req.(type) {
+	case *csi.NodeStageVolumeRequest:
+		return vr.GetStagingTargetPath() + "/" + vr.GetVolumeId()
+	case *csi.NodeUnstageVolumeRequest:
+		return vr.GetStagingTargetPath() + "/" + vr.GetVolumeId()
+	}
+
+	return ""
+}
+
+// NodeUnstageVolume unstages the volume from the staging path
+func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	var err error
+	if err = util.ValidateNodeUnstageVolumeRequest(req); err != nil {
+		return nil, err
+	}
+
+	volID := req.GetVolumeId()
+
+	if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ns.VolumeLocks.Release(volID)
+
+	stagingParentPath := req.GetStagingTargetPath()
+	stagingTargetPath := getStagingTargetPath(req)
+
+	notMnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, status.Error(codes.NotFound, err.Error())
+		}
+		// Continue on ENOENT errors as we may still have the image mapped
+		notMnt = true
+	}
+	if !notMnt {
+		// Unmounting the image
+		err = ns.mounter.Unmount(stagingTargetPath)
+		if err != nil {
+			klog.V(3).Infof(util.Log(ctx, "failed to unmount targetPath: %s with error: %v"), stagingTargetPath, err)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	if err = os.Remove(stagingTargetPath); err != nil {
+		// Any error is critical as Staging path is expected to be empty by Kubernetes, it otherwise
+		// keeps invoking Unstage. Hence any errors removing files within this path is a critical
+		// error
+		if !os.IsNotExist(err) {
+			klog.Errorf(util.Log(ctx, "failed to remove staging target path (%s): (%v)"), stagingTargetPath, err)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
+	imgInfo, err := lookupRBDImageMetadataStash(stagingParentPath)
+	if err != nil {
+		klog.V(2).Infof(util.Log(ctx, "failed to find image metadata: %v"), err)
+		// It is an error if it was mounted, as we should have found the image metadata file with
+		// no errors
+		if !notMnt {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// If not mounted, and error is anything other than metadata file missing, it is an error
+		if _, ok := err.(ErrMissingStash); !ok {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		// It was not mounted and image metadata is also missing, we are done as the last step in
+		// in the staging transaction is complete
+		return &csi.NodeUnstageVolumeResponse{}, nil
+	}
+
+	// Unmapping rbd device
+	imageSpec := imgInfo.Pool + "/" + imgInfo.ImageName
+	if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess, imgInfo.Encrypted, req.GetVolumeId()); err != nil {
+		klog.Errorf(util.Log(ctx, "error unmapping volume (%s) from staging path (%s): (%v)"), req.GetVolumeId(), stagingTargetPath, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "successfully unmounted volume (%s) from staging path (%s)"),
+		req.GetVolumeId(), stagingTargetPath)
+
+	if err = cleanupRBDImageMetadataStash(stagingParentPath); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to cleanup image metadata stash (%v)"), err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	return &csi.NodeUnstageVolumeResponse{}, nil
+}
+
+// NodeExpandVolume resizes rbd volumes
+func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
+	volumeID := req.GetVolumeId()
+	if volumeID == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume ID must be provided")
+	}
+	volumePath := req.GetVolumePath()
+	if volumePath == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume path must be provided")
+	}
+
+	if acquired := ns.VolumeLocks.TryAcquire(volumeID); !acquired {
+		klog.Errorf(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volumeID)
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID)
+	}
+	defer ns.VolumeLocks.Release(volumeID)
+
+	// volumePath is targetPath for block PVC and stagingPath for filesystem.
+	// check the path is mountpoint or not, if it is
+	// mountpoint treat this as block PVC or else it is filesystem PVC
+	// TODO remove this once ceph-csi supports CSI v1.2.0 spec
+	notMnt, err := mount.IsNotMountPoint(ns.mounter, volumePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, status.Error(codes.NotFound, err.Error())
+		}
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	if !notMnt {
+		return &csi.NodeExpandVolumeResponse{}, nil
+	}
+
+	devicePath, err := getDevicePath(ctx, volumePath)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+	// TODO check size and return success or error
+	volumePath += "/" + volumeID
+	resizer := resizefs.NewResizeFs(diskMounter)
+	ok, err := resizer.Resize(devicePath, volumePath)
+	if !ok {
+		return nil, fmt.Errorf("rbd: resize failed on path %s, error: %v", req.GetVolumePath(), err)
+	}
+	return &csi.NodeExpandVolumeResponse{}, nil
+}
+
+func getDevicePath(ctx context.Context, volumePath string) (string, error) {
+	imgInfo, err := lookupRBDImageMetadataStash(volumePath)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to find image metadata: %v"), err)
+	}
+	device, found := findDeviceMappingImage(ctx, imgInfo.Pool, imgInfo.ImageName, imgInfo.NbdAccess)
+	if found {
+		return device, nil
+	}
+	return "", fmt.Errorf("failed to get device for stagingtarget path %v", volumePath)
+}
+
+// NodeGetCapabilities returns the supported capabilities of the node server
+func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
+	return &csi.NodeGetCapabilitiesResponse{
+		Capabilities: []*csi.NodeServiceCapability{
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
+					},
+				},
+			},
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS,
+					},
+				},
+			},
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials) (string, error) {
+	imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName
+	encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"),
+			imageSpec, err)
+		return "", err
+	}
+
+	if encrypted == rbdImageRequiresEncryption {
+		diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+		// TODO: update this when adding support for static (pre-provisioned) PVs
+		var existingFormat string
+		existingFormat, err = diskMounter.GetDiskFormat(devicePath)
+		if err != nil {
+			return "", fmt.Errorf("failed to get disk format for path %s, error: %v", devicePath, err)
+		}
+
+		switch existingFormat {
+		case "":
+			err = encryptDevice(ctx, volOptions, cr, devicePath)
+			if err != nil {
+				return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err)
+			}
+		case "crypt":
+			klog.Warningf(util.Log(ctx, "rbd image %s is encrypted, but encryption state was not updated"),
+				imageSpec)
+			err = util.SaveRbdImageEncryptionStatus(
+				ctx, cr, volOptions.Monitors, imageSpec, rbdImageEncrypted)
+			if err != nil {
+				return "", fmt.Errorf("failed to update encryption state for rbd image %s", imageSpec)
+			}
+		default:
+			return "", fmt.Errorf("can not encrypt rbdImage %s that already has file system: %s",
+				imageSpec, existingFormat)
+		}
+	} else if encrypted != rbdImageEncrypted {
+		return "", fmt.Errorf("rbd image %s found mounted with unexpected encryption status %s",
+			imageSpec, encrypted)
+	}
+
+	devicePath, err = openEncryptedDevice(ctx, volOptions, devicePath)
+	if err != nil {
+		return "", err
+	}
+
+	return devicePath, nil
+}
+
+func encryptDevice(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials, devicePath string) error {
+	passphrase, err := util.GetCryptoPassphrase(ctx, rbdVol.VolID, rbdVol.KMS)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"),
+			rbdVol.Pool, rbdVol.RbdImageName, err)
+		return err
+	}
+
+	if err = util.EncryptVolume(ctx, devicePath, passphrase); err != nil {
+		err = fmt.Errorf("failed to encrypt volume %s/%s: %v", rbdVol.Pool, rbdVol.RbdImageName, err)
+		klog.Errorf(util.Log(ctx, err.Error()))
+		return err
+	}
+
+	imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName
+	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted)
+
+	return err
+}
+
+func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string) (string, error) {
+	passphrase, err := util.GetCryptoPassphrase(ctx, volOptions.VolID, volOptions.KMS)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get passphrase for encrypted device %s/%s: %v"),
+			volOptions.Pool, volOptions.RbdImageName, err)
+		return "", status.Error(codes.Internal, err.Error())
+	}
+
+	mapperFile, mapperFilePath := util.VolumeMapper(volOptions.VolID)
+
+	isOpen, err := util.IsDeviceOpen(ctx, mapperFilePath)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to check device %s encryption status: %s"), devicePath, err)
+		return devicePath, err
+	}
+	if isOpen {
+		klog.V(4).Infof(util.Log(ctx, "encrypted device is already open at %s"), mapperFilePath)
+	} else {
+		err = util.OpenEncryptedVolume(ctx, devicePath, mapperFile, passphrase)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to open device %s/%s: %v"),
+				volOptions.Pool, volOptions.RbdImageName, err)
+			return devicePath, err
+		}
+	}
+
+	return mapperFilePath, nil
+}

internal/rbd/nodeserver_test.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"testing"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+func TestGetLegacyVolumeName(t *testing.T) {
+	tests := []struct {
+		mountPath string
+		volName   string
+	}{
+		{"csi/vol/56a0cc34-a5c9-44ab-ad33-ed53dd2bd5ea/globalmount", "56a0cc34-a5c9-44ab-ad33-ed53dd2bd5ea"},
+		{"csi/vol/9fdb7491-3469-4414-8fe2-ea96be6f7f72/mount", "9fdb7491-3469-4414-8fe2-ea96be6f7f72"},
+		{"csi/vol/82cd91c4-4582-47b3-bb08-a84f8c5716d6", "82cd91c4-4582-47b3-bb08-a84f8c5716d6"},
+	}
+
+	for _, test := range tests {
+		if got, err := getLegacyVolumeName(test.mountPath); err != nil {
+			t.Errorf("getLegacyVolumeName(%s) returned error when it shouldn't: %s", test.mountPath, err.Error())
+		} else if got != test.volName {
+			t.Errorf("getLegacyVolumeName(%s) = %s, want %s", test.mountPath, got, test.volName)
+		}
+	}
+}
+
+func TestGetStagingPath(t *testing.T) {
+	var stagingPath string
+	// test with nodestagevolumerequest
+	nsvr := &csi.NodeStageVolumeRequest{
+		VolumeId:          "758978be-6331-4925-b25e-e490fe99c9eb",
+		StagingTargetPath: "/path/to/stage",
+	}
+
+	expect := "/path/to/stage/758978be-6331-4925-b25e-e490fe99c9eb"
+	stagingPath = getStagingTargetPath(nsvr)
+	if stagingPath != expect {
+		t.Errorf("getStagingTargetPath() = %s, got %s", stagingPath, expect)
+	}
+
+	// test with nodestagevolumerequest
+	nuvr := &csi.NodeUnstageVolumeRequest{
+		VolumeId:          "622cfdeb-69bf-4de6-9bd7-5fa0b71a603e",
+		StagingTargetPath: "/path/to/unstage",
+	}
+
+	expect = "/path/to/unstage/622cfdeb-69bf-4de6-9bd7-5fa0b71a603e"
+	stagingPath = getStagingTargetPath(nuvr)
+	if stagingPath != expect {
+		t.Errorf("getStagingTargetPath() = %s, got %s", stagingPath, expect)
+	}
+
+	// test with non-handled interface
+	expect = ""
+	stagingPath = getStagingTargetPath("")
+	if stagingPath != expect {
+		t.Errorf("getStagingTargetPath() = %s, got %s", stagingPath, expect)
+	}
+}

internal/rbd/rbd_attach.go

@@ -0,0 +1,324 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
+)
+
+const (
+	rbdTonbd  = "rbd-nbd"
+	moduleNbd = "nbd"
+
+	accessTypeKRbd = "krbd"
+	accessTypeNbd  = "nbd"
+
+	rbd = "rbd"
+
+	// Output strings returned during invocation of "rbd unmap --device-type... <imageSpec>" when
+	// image is not found to be mapped. Used to ignore errors when attempting to unmap such images.
+	// The %s format specifier should contain the <imageSpec> string
+	// NOTE: When using devicePath instead of imageSpec, the error strings are different
+	rbdUnmapCmdkRbdMissingMap = "rbd: %s: not a mapped image or snapshot"
+	rbdUnmapCmdNbdMissingMap  = "rbd-nbd: %s is not mapped"
+	rbdMapConnectionTimeout   = "Connection timed out"
+)
+
+var hasNBD = false
+
+func init() {
+	hasNBD = checkRbdNbdTools()
+}
+
+// rbdDeviceInfo strongly typed JSON spec for rbd device list output (of type krbd)
+type rbdDeviceInfo struct {
+	ID     string `json:"id"`
+	Pool   string `json:"pool"`
+	Name   string `json:"name"`
+	Device string `json:"device"`
+}
+
+// nbdDeviceInfo strongly typed JSON spec for rbd-nbd device list output (of type nbd)
+// NOTE: There is a bug in rbd output that returns id as number for nbd, and string for krbd, thus
+// requiring 2 different JSON structures to unmarshal the output.
+// NOTE: image key is "name" in krbd output and "image" in nbd output, which is another difference
+type nbdDeviceInfo struct {
+	ID     int64  `json:"id"`
+	Pool   string `json:"pool"`
+	Name   string `json:"image"`
+	Device string `json:"device"`
+}
+
+// rbdGetDeviceList queries rbd about mapped devices and returns a list of rbdDeviceInfo
+// It will selectively list devices mapped using krbd or nbd as specified by accessType
+func rbdGetDeviceList(accessType string) ([]rbdDeviceInfo, error) {
+	// rbd device list --format json --device-type [krbd|nbd]
+	var (
+		rbdDeviceList []rbdDeviceInfo
+		nbdDeviceList []nbdDeviceInfo
+	)
+
+	stdout, _, err := util.ExecCommand(rbd, "device", "list", "--format="+"json", "--device-type", accessType)
+	if err != nil {
+		return nil, fmt.Errorf("error getting device list from rbd for devices of type (%s): (%v)", accessType, err)
+	}
+
+	if accessType == accessTypeKRbd {
+		err = json.Unmarshal(stdout, &rbdDeviceList)
+	} else {
+		err = json.Unmarshal(stdout, &nbdDeviceList)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error to parse JSON output of device list for devices of type (%s): (%v)", accessType, err)
+	}
+
+	// convert output to a rbdDeviceInfo list for consumers
+	if accessType == accessTypeNbd {
+		for _, device := range nbdDeviceList {
+			rbdDeviceList = append(
+				rbdDeviceList,
+				rbdDeviceInfo{
+					ID:     strconv.FormatInt(device.ID, 10),
+					Pool:   device.Pool,
+					Name:   device.Name,
+					Device: device.Device,
+				})
+		}
+	}
+
+	return rbdDeviceList, nil
+}
+
+// findDeviceMappingImage finds a devicePath, if available, based on image spec (pool/image) on the node.
+func findDeviceMappingImage(ctx context.Context, pool, image string, useNbdDriver bool) (string, bool) {
+	accessType := accessTypeKRbd
+	if useNbdDriver {
+		accessType = accessTypeNbd
+	}
+
+	rbdDeviceList, err := rbdGetDeviceList(accessType)
+	if err != nil {
+		klog.Warningf(util.Log(ctx, "failed to determine if image (%s/%s) is mapped to a device (%v)"), pool, image, err)
+		return "", false
+	}
+
+	for _, device := range rbdDeviceList {
+		if device.Name == image && device.Pool == pool {
+			return device.Device, true
+		}
+	}
+
+	return "", false
+}
+
+// Stat a path, if it doesn't exist, retry maxRetries times.
+func waitForPath(ctx context.Context, pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
+	for i := 0; i < maxRetries; i++ {
+		if i != 0 {
+			time.Sleep(time.Second)
+		}
+
+		device, found := findDeviceMappingImage(ctx, pool, image, useNbdDriver)
+		if found {
+			return device, found
+		}
+	}
+
+	return "", false
+}
+
+// Check if rbd-nbd tools are installed.
+func checkRbdNbdTools() bool {
+	// check if the module is loaded or compiled in
+	_, err := os.Stat(fmt.Sprintf("/sys/module/%s", moduleNbd))
+	if os.IsNotExist(err) {
+		// try to load the module
+		_, err = execCommand("modprobe", []string{moduleNbd})
+		if err != nil {
+			klog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
+			return false
+		}
+	}
+	if _, err := execCommand(rbdTonbd, []string{"--version"}); err != nil {
+		klog.V(3).Infof("rbd-nbd: running rbd-nbd --version failed with error %v", err)
+		return false
+	}
+	klog.V(3).Infof("rbd-nbd tools were found.")
+	return true
+}
+
+func attachRBDImage(ctx context.Context, volOptions *rbdVolume, cr *util.Credentials) (string, error) {
+	var err error
+
+	image := volOptions.RbdImageName
+	useNBD := false
+	if volOptions.Mounter == rbdTonbd && hasNBD {
+		useNBD = true
+	}
+
+	devicePath, found := waitForPath(ctx, volOptions.Pool, image, 1, useNBD)
+	if !found {
+		backoff := wait.Backoff{
+			Duration: rbdImageWatcherInitDelay,
+			Factor:   rbdImageWatcherFactor,
+			Steps:    rbdImageWatcherSteps,
+		}
+
+		err = waitForrbdImage(ctx, backoff, volOptions, cr)
+
+		if err != nil {
+			return "", err
+		}
+		devicePath, err = createPath(ctx, volOptions, cr)
+	}
+
+	return devicePath, err
+}
+
+func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (string, error) {
+	isNbd := false
+	image := volOpt.RbdImageName
+	imagePath := fmt.Sprintf("%s/%s", volOpt.Pool, image)
+
+	klog.V(5).Infof(util.Log(ctx, "rbd: map mon %s"), volOpt.Monitors)
+
+	// Map options
+	mapOptions := []string{
+		"--id", cr.ID,
+		"-m", volOpt.Monitors,
+		"--keyfile=" + cr.KeyFile,
+		"map", imagePath,
+	}
+
+	// Choose access protocol
+	accessType := accessTypeKRbd
+	if volOpt.Mounter == rbdTonbd && hasNBD {
+		isNbd = true
+		accessType = accessTypeNbd
+	}
+
+	// Update options with device type selection
+	mapOptions = append(mapOptions, "--device-type", accessType)
+
+	// Execute map
+	output, err := execCommand(rbd, mapOptions)
+	if err != nil {
+		klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output))
+		// unmap rbd image if connection timeout
+		if strings.Contains(err.Error(), rbdMapConnectionTimeout) {
+			detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.Encrypted, volOpt.VolID)
+			if detErr != nil {
+				klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr)
+			}
+		}
+		return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
+	}
+	devicePath := strings.TrimSuffix(string(output), "\n")
+
+	return devicePath, nil
+}
+
+func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdVolume, cr *util.Credentials) error {
+	image := volOptions.RbdImageName
+	imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image)
+
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		used, rbdOutput, err := rbdStatus(ctx, volOptions, cr)
+		if err != nil {
+			return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
+		}
+		if (volOptions.DisableInUseChecks) && (used) {
+			klog.V(2).Info(util.Log(ctx, "valid multi-node attach requested, ignoring watcher in-use result"))
+			return used, nil
+		}
+		return !used, nil
+	})
+	// return error if rbd image has not become available for the specified timeout
+	if err == wait.ErrWaitTimeout {
+		return fmt.Errorf("rbd image %s is still being used", imagePath)
+	}
+	// return error if any other errors were encountered during waiting for the image to become available
+	return err
+}
+
+func detachRBDDevice(ctx context.Context, devicePath, volumeID string, encrypted bool) error {
+	nbdType := false
+	if strings.HasPrefix(devicePath, "/dev/nbd") {
+		nbdType = true
+	}
+
+	return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, encrypted, volumeID)
+}
+
+// detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking
+// when imageSpec is used to decide if image is already unmapped
+func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType, encrypted bool, volumeID string) error {
+	var output []byte
+
+	if encrypted {
+		mapperFile, mapperPath := util.VolumeMapper(volumeID)
+		mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"),
+				mapperPath, imageOrDeviceSpec, err)
+			return err
+		}
+		if len(mapper) > 0 {
+			// mapper found, so it is open Luks device
+			err = util.CloseEncryptedVolume(ctx, mapperFile)
+			if err != nil {
+				klog.Errorf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"),
+					mapperPath, imageOrDeviceSpec, err)
+				return err
+			}
+			imageOrDeviceSpec = mappedDevice
+		}
+	}
+
+	accessType := accessTypeKRbd
+	if ndbType {
+		accessType = accessTypeNbd
+	}
+	options := []string{"unmap", "--device-type", accessType, imageOrDeviceSpec}
+
+	output, err := execCommand(rbd, options)
+	if err != nil {
+		// Messages for krbd and nbd differ, hence checking either of them for missing mapping
+		// This is not applicable when a device path is passed in
+		if isImageSpec &&
+			(strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdkRbdMissingMap, imageOrDeviceSpec)) ||
+				strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdNbdMissingMap, imageOrDeviceSpec))) {
+			// Devices found not to be mapped are treated as a successful detach
+			klog.V(5).Infof(util.Log(ctx, "image or device spec (%s) not mapped"), imageOrDeviceSpec)
+			return nil
+		}
+		return fmt.Errorf("rbd: unmap for spec (%s) failed (%v): (%s)", imageOrDeviceSpec, err, string(output))
+	}
+
+	return nil
+}

internal/rbd/rbd_journal.go

@@ -0,0 +1,353 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog"
+)
+
+func validateNonEmptyField(field, fieldName, structName string) error {
+	if field == "" {
+		return fmt.Errorf("value '%s' in '%s' structure cannot be empty", fieldName, structName)
+	}
+
+	return nil
+}
+
+func validateRbdSnap(rbdSnap *rbdSnapshot) error {
+	var err error
+
+	if err = validateNonEmptyField(rbdSnap.RequestName, "RequestName", "rbdSnapshot"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdSnap.Monitors, "Monitors", "rbdSnapshot"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdSnap.Pool, "Pool", "rbdSnapshot"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdSnap.RbdImageName, "RbdImageName", "rbdSnapshot"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdSnap.ClusterID, "ClusterID", "rbdSnapshot"); err != nil {
+		return err
+	}
+
+	return err
+}
+
+func validateRbdVol(rbdVol *rbdVolume) error {
+	var err error
+
+	if err = validateNonEmptyField(rbdVol.RequestName, "RequestName", "rbdVolume"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdVol.Monitors, "Monitors", "rbdVolume"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdVol.Pool, "Pool", "rbdVolume"); err != nil {
+		return err
+	}
+
+	if err = validateNonEmptyField(rbdVol.ClusterID, "ClusterID", "rbdVolume"); err != nil {
+		return err
+	}
+
+	if rbdVol.VolSize == 0 {
+		return errors.New("value 'VolSize' in 'rbdVolume' structure cannot be 0")
+	}
+
+	return err
+}
+
+/*
+checkSnapExists, and its counterpart checkVolExists, function checks if the passed in rbdSnapshot
+or rbdVolume exists on the backend.
+
+**NOTE:** These functions manipulate the rados omaps that hold information regarding
+volume names as requested by the CSI drivers. Hence, these need to be invoked only when the
+respective CSI driver generated snapshot or volume name based locks are held, as otherwise racy
+access to these omaps may end up leaving them in an inconsistent state.
+
+These functions need enough information about cluster and pool (ie, Monitors, Pool, IDs filled in)
+to operate. They further require that the RequestName element of the structure have a valid value
+to operate on and determine if the said RequestName already exists on the backend.
+
+These functions populate the snapshot or the image name, its attributes and the CSI snapshot/volume
+ID for the same when successful.
+
+These functions also cleanup omap reservations that are stale. I.e when omap entries exist and
+backing images or snapshots are missing, or one of the omaps exist and the next is missing. This is
+because, the order of omap creation and deletion are inverse of each other, and protected by the
+request name lock, and hence any stale omaps are leftovers from incomplete transactions and are
+hence safe to garbage collect.
+*/
+func checkSnapExists(ctx context.Context, rbdSnap *rbdSnapshot, cr *util.Credentials) (bool, error) {
+	err := validateRbdSnap(rbdSnap)
+	if err != nil {
+		return false, err
+	}
+
+	snapData, err := snapJournal.CheckReservation(ctx, rbdSnap.Monitors, cr, rbdSnap.JournalPool,
+		rbdSnap.RequestName, rbdSnap.NamePrefix, rbdSnap.RbdImageName, "")
+	if err != nil {
+		return false, err
+	}
+	if snapData == nil {
+		return false, nil
+	}
+	snapUUID := snapData.ImageUUID
+	rbdSnap.RbdSnapName = snapData.ImageAttributes.ImageName
+
+	// it should never happen that this disagrees, but check
+	if rbdSnap.Pool != snapData.ImagePool {
+		return false, fmt.Errorf("stored snapshot pool (%s) and expected snapshot pool (%s) mismatch",
+			snapData.ImagePool, rbdSnap.Pool)
+	}
+
+	// Fetch on-disk image attributes
+	err = updateSnapWithImageInfo(ctx, rbdSnap, cr)
+	if err != nil {
+		if _, ok := err.(ErrSnapNotFound); ok {
+			err = snapJournal.UndoReservation(ctx, rbdSnap.Monitors, cr, rbdSnap.JournalPool,
+				rbdSnap.Pool, rbdSnap.RbdSnapName, rbdSnap.RequestName)
+			return false, err
+		}
+		return false, err
+	}
+
+	// found a snapshot already available, process and return its information
+	rbdSnap.SnapID, err = util.GenerateVolID(ctx, rbdSnap.Monitors, cr, snapData.ImagePoolID, rbdSnap.Pool,
+		rbdSnap.ClusterID, snapUUID, volIDVersion)
+	if err != nil {
+		return false, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "found existing snap (%s) with snap name (%s) for request (%s)"),
+		rbdSnap.SnapID, rbdSnap.RbdSnapName, rbdSnap.RequestName)
+
+	return true, nil
+}
+
+/*
+Check comment on checkSnapExists, to understand how this function behaves
+
+**NOTE:** These functions manipulate the rados omaps that hold information regarding
+volume names as requested by the CSI drivers. Hence, these need to be invoked only when the
+respective CSI snapshot or volume name based locks are held, as otherwise racy access to these
+omaps may end up leaving the omaps in an inconsistent state.
+*/
+func checkVolExists(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) (bool, error) {
+	err := validateRbdVol(rbdVol)
+	if err != nil {
+		return false, err
+	}
+
+	kmsID := ""
+	if rbdVol.Encrypted {
+		kmsID = rbdVol.KMS.GetID()
+	}
+
+	imageData, err := volJournal.CheckReservation(ctx, rbdVol.Monitors, cr, rbdVol.JournalPool,
+		rbdVol.RequestName, rbdVol.NamePrefix, "", kmsID)
+	if err != nil {
+		return false, err
+	}
+	if imageData == nil {
+		return false, nil
+	}
+
+	imageUUID := imageData.ImageUUID
+	rbdVol.RbdImageName = imageData.ImageAttributes.ImageName
+
+	// check if topology constraints match what is found
+	rbdVol.Topology, err = util.MatchTopologyForPool(rbdVol.TopologyPools,
+		rbdVol.TopologyRequirement, imageData.ImagePool)
+	if err != nil {
+		// TODO check if need any undo operation here, or ErrVolNameConflict
+		return false, err
+	}
+	// update Pool, if it was topology constrained
+	if rbdVol.Topology != nil {
+		rbdVol.Pool = imageData.ImagePool
+	}
+
+	// NOTE: Return volsize should be on-disk volsize, not request vol size, so
+	// save it for size checks before fetching image data
+	requestSize := rbdVol.VolSize
+	// Fetch on-disk image attributes and compare against request
+	err = updateVolWithImageInfo(ctx, rbdVol, cr)
+	if err != nil {
+		if _, ok := err.(ErrImageNotFound); ok {
+			err = volJournal.UndoReservation(ctx, rbdVol.Monitors, cr, rbdVol.JournalPool, rbdVol.Pool,
+				rbdVol.RbdImageName, rbdVol.RequestName)
+			return false, err
+		}
+		return false, err
+	}
+
+	// size checks
+	if rbdVol.VolSize < requestSize {
+		err = fmt.Errorf("image with the same name (%s) but with different size already exists",
+			rbdVol.RbdImageName)
+		return false, ErrVolNameConflict{rbdVol.RbdImageName, err}
+	}
+	// TODO: We should also ensure image features and format is the same
+
+	// found a volume already available, process and return it!
+	rbdVol.VolID, err = util.GenerateVolID(ctx, rbdVol.Monitors, cr, imageData.ImagePoolID, rbdVol.Pool,
+		rbdVol.ClusterID, imageUUID, volIDVersion)
+	if err != nil {
+		return false, err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "found existing volume (%s) with image name (%s) for request (%s)"),
+		rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName)
+
+	return true, nil
+}
+
+// reserveSnap is a helper routine to request a rbdSnapshot name reservation and generate the
+// volume ID for the generated name
+func reserveSnap(ctx context.Context, rbdSnap *rbdSnapshot, cr *util.Credentials) error {
+	var (
+		snapUUID string
+		err      error
+	)
+
+	journalPoolID, imagePoolID, err := util.GetPoolIDs(ctx, rbdSnap.Monitors, rbdSnap.JournalPool, rbdSnap.Pool, cr)
+	if err != nil {
+		return err
+	}
+
+	snapUUID, rbdSnap.RbdSnapName, err = snapJournal.ReserveName(ctx, rbdSnap.Monitors, cr, rbdSnap.JournalPool, journalPoolID,
+		rbdSnap.Pool, imagePoolID, rbdSnap.RequestName, rbdSnap.NamePrefix, rbdSnap.RbdImageName, "")
+	if err != nil {
+		return err
+	}
+
+	rbdSnap.SnapID, err = util.GenerateVolID(ctx, rbdSnap.Monitors, cr, imagePoolID, rbdSnap.Pool,
+		rbdSnap.ClusterID, snapUUID, volIDVersion)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)"),
+		rbdSnap.SnapID, rbdSnap.RbdSnapName, rbdSnap.RequestName)
+
+	return nil
+}
+
+func updateTopologyConstraints(rbdVol *rbdVolume, rbdSnap *rbdSnapshot) error {
+	var err error
+	if rbdSnap != nil {
+		// check if topology constraints matches snapshot pool
+		rbdVol.Topology, err = util.MatchTopologyForPool(rbdVol.TopologyPools,
+			rbdVol.TopologyRequirement, rbdSnap.Pool)
+		if err != nil {
+			return err
+		}
+
+		// update Pool, if it was topology constrained
+		if rbdVol.Topology != nil {
+			rbdVol.Pool = rbdSnap.Pool
+		}
+
+		return nil
+	}
+	// update request based on topology constrained parameters (if present)
+	poolName, dataPoolName, topology, err := util.FindPoolAndTopology(rbdVol.TopologyPools, rbdVol.TopologyRequirement)
+	if err != nil {
+		return err
+	}
+	if poolName != "" {
+		rbdVol.Pool = poolName
+		rbdVol.DataPool = dataPoolName
+		rbdVol.Topology = topology
+	}
+
+	return nil
+}
+
+// reserveVol is a helper routine to request a rbdVolume name reservation and generate the
+// volume ID for the generated name
+func reserveVol(ctx context.Context, rbdVol *rbdVolume, rbdSnap *rbdSnapshot, cr *util.Credentials) error {
+	var (
+		imageUUID string
+		err       error
+	)
+
+	err = updateTopologyConstraints(rbdVol, rbdSnap)
+	if err != nil {
+		return err
+	}
+
+	journalPoolID, imagePoolID, err := util.GetPoolIDs(ctx, rbdVol.Monitors, rbdVol.JournalPool, rbdVol.Pool, cr)
+	if err != nil {
+		return err
+	}
+
+	kmsID := ""
+	if rbdVol.Encrypted {
+		kmsID = rbdVol.KMS.GetID()
+	}
+
+	imageUUID, rbdVol.RbdImageName, err = volJournal.ReserveName(ctx, rbdVol.Monitors, cr, rbdVol.JournalPool, journalPoolID,
+		rbdVol.Pool, imagePoolID, rbdVol.RequestName, rbdVol.NamePrefix, "", kmsID)
+	if err != nil {
+		return err
+	}
+
+	rbdVol.VolID, err = util.GenerateVolID(ctx, rbdVol.Monitors, cr, imagePoolID, rbdVol.Pool,
+		rbdVol.ClusterID, imageUUID, volIDVersion)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)"),
+		rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName)
+
+	return nil
+}
+
+// undoSnapReservation is a helper routine to undo a name reservation for rbdSnapshot
+func undoSnapReservation(ctx context.Context, rbdSnap *rbdSnapshot, cr *util.Credentials) error {
+	err := snapJournal.UndoReservation(ctx, rbdSnap.Monitors, cr, rbdSnap.JournalPool, rbdSnap.Pool,
+		rbdSnap.RbdSnapName, rbdSnap.RequestName)
+
+	return err
+}
+
+// undoVolReservation is a helper routine to undo a name reservation for rbdVolume
+func undoVolReservation(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
+	err := volJournal.UndoReservation(ctx, rbdVol.Monitors, cr, rbdVol.JournalPool, rbdVol.Pool,
+		rbdVol.RbdImageName, rbdVol.RequestName)
+
+	return err
+}

internal/rbd/rbd_util.go

@@ -0,0 +1,999 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/ceph/go-ceph/rados"
+	librbd "github.com/ceph/go-ceph/rbd"
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/golang/protobuf/ptypes/timestamp"
+	"github.com/pborman/uuid"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/cloud-provider/volume/helpers"
+	"k8s.io/klog"
+)
+
+const (
+	imageWatcherStr = "watcher="
+	// The following three values are used for 30 seconds timeout
+	// while waiting for RBD Watcher to expire.
+	rbdImageWatcherInitDelay = 1 * time.Second
+	rbdImageWatcherFactor    = 1.4
+	rbdImageWatcherSteps     = 10
+	rbdDefaultMounter        = "rbd"
+
+	// Output strings returned during invocation of "ceph rbd task add remove <imagespec>" when
+	// command is not supported by ceph manager. Used to check errors and recover when the command
+	// is unsupported.
+	rbdTaskRemoveCmdInvalidString1      = "no valid command found"
+	rbdTaskRemoveCmdInvalidString2      = "Error EINVAL: invalid command"
+	rbdTaskRemoveCmdAccessDeniedMessage = "Error EACCES:"
+
+	// Encryption statuses for RbdImage
+	rbdImageEncrypted          = "encrypted"
+	rbdImageRequiresEncryption = "requiresEncryption"
+)
+
+// rbdVolume represents a CSI volume and its RBD image specifics
+type rbdVolume struct {
+	// RbdImageName is the name of the RBD image backing this rbdVolume. This does not have a
+	//   JSON tag as it is not stashed in JSON encoded config maps in v1.0.0
+	// VolID is the volume ID that is exchanged with CSI drivers, identifying this rbdVol
+	// RequestName is the CSI generated volume name for the rbdVolume.  This does not have a
+	//   JSON tag as it is not stashed in JSON encoded config maps in v1.0.0
+	// VolName and MonValueFromSecret are retained from older plugin versions (<= 1.0.0)
+	//   for backward compatibility reasons
+	// JournalPool is the ceph pool in which the CSI Journal is stored
+	// Pool is where the image journal and image is stored, and could be the same as `JournalPool`
+	//   (retained as Pool instead of renaming to ImagePool or such, as this is referenced in the code extensively)
+	// DataPool is where the data for images in `Pool` are stored, this is used as the `--data-pool`
+	// 	 argument when the pool is created, and is not used anywhere else
+	TopologyPools       *[]util.TopologyConstrainedPool
+	TopologyRequirement *csi.TopologyRequirement
+	Topology            map[string]string
+	RbdImageName        string
+	NamePrefix          string
+	VolID               string `json:"volID"`
+	Monitors            string `json:"monitors"`
+	JournalPool         string
+	Pool                string `json:"pool"`
+	DataPool            string
+	ImageFeatures       string `json:"imageFeatures"`
+	AdminID             string `json:"adminId"`
+	UserID              string `json:"userId"`
+	Mounter             string `json:"mounter"`
+	ClusterID           string `json:"clusterId"`
+	RequestName         string
+	VolName             string `json:"volName"`
+	MonValueFromSecret  string `json:"monValueFromSecret"`
+	VolSize             int64  `json:"volSize"`
+	DisableInUseChecks  bool   `json:"disableInUseChecks"`
+	Encrypted           bool
+	KMS                 util.EncryptionKMS
+
+	// connection
+	conn *rados.Conn
+}
+
+// rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics
+type rbdSnapshot struct {
+	// SourceVolumeID is the volume ID of RbdImageName, that is exchanged with CSI drivers
+	// RbdImageName is the name of the RBD image, that is this rbdSnapshot's source image
+	// RbdSnapName is the name of the RBD snapshot backing this rbdSnapshot
+	// SnapID is the snapshot ID that is exchanged with CSI drivers, identifying this rbdSnapshot
+	// RequestName is the CSI generated snapshot name for the rbdSnapshot
+	// JournalPool is the ceph pool in which the CSI snapshot Journal is stored
+	// Pool is where the image snapshot journal and snapshot is stored, and could be the same as `JournalPool`
+	SourceVolumeID string
+	RbdImageName   string
+	NamePrefix     string
+	RbdSnapName    string
+	SnapID         string
+	Monitors       string
+	JournalPool    string
+	Pool           string
+	CreatedAt      *timestamp.Timestamp
+	SizeBytes      int64
+	ClusterID      string
+	RequestName    string
+}
+
+var (
+	supportedFeatures = sets.NewString("layering")
+
+	// large interval and timeout, it should be longer than the maximum
+	// time an operation can take (until refcounting of the connections is
+	// available)
+	cpInterval = 15 * time.Minute
+	cpExpiry   = 10 * time.Minute
+	connPool   = util.NewConnPool(cpInterval, cpExpiry)
+)
+
+// createImage creates a new ceph image with provision and volume options.
+func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) error {
+	volSzMiB := fmt.Sprintf("%dM", util.RoundOffVolSize(pOpts.VolSize))
+	options := librbd.NewRbdImageOptions()
+
+	logMsg := "rbd: create %s size %s (features: %s) using mon %s, pool %s "
+	if pOpts.DataPool != "" {
+		logMsg += fmt.Sprintf("data pool %s", pOpts.DataPool)
+		err := options.SetString(librbd.RbdImageOptionDataPool, pOpts.DataPool)
+		if err != nil {
+			return errors.Wrapf(err, "failed to set data pool")
+		}
+	}
+	klog.V(4).Infof(util.Log(ctx, logMsg),
+		pOpts.RbdImageName, volSzMiB, pOpts.ImageFeatures, pOpts.Monitors, pOpts.Pool)
+
+	if pOpts.ImageFeatures != "" {
+		features := imageFeaturesToUint64(ctx, pOpts.ImageFeatures)
+		err := options.SetUint64(librbd.RbdImageOptionFeatures, features)
+		if err != nil {
+			return errors.Wrapf(err, "failed to set image features")
+		}
+	}
+
+	ioctx, err := pOpts.getIoctx(cr)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get IOContext")
+	}
+	defer ioctx.Destroy()
+
+	err = librbd.CreateImage(ioctx, pOpts.RbdImageName,
+		uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create rbd image")
+	}
+
+	return nil
+}
+
+func (rv *rbdVolume) getIoctx(cr *util.Credentials) (*rados.IOContext, error) {
+	if rv.conn == nil {
+		conn, err := connPool.Get(rv.Pool, rv.Monitors, cr.ID, cr.KeyFile)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get connection")
+		}
+
+		rv.conn = conn
+	}
+
+	ioctx, err := rv.conn.OpenIOContext(rv.Pool)
+	if err != nil {
+		connPool.Put(rv.conn)
+		return nil, errors.Wrapf(err, "failed to open IOContext for pool %s", rv.Pool)
+	}
+
+	return ioctx, nil
+}
+
+func (rv *rbdVolume) Destroy() {
+	if rv.conn != nil {
+		connPool.Put(rv.conn)
+	}
+}
+
+// rbdStatus checks if there is watcher on the image.
+// It returns true if there is a watcher on the image, otherwise returns false.
+func rbdStatus(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) (bool, string, error) {
+	var output string
+	var cmd []byte
+
+	image := pOpts.RbdImageName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: status %s using mon %s, pool %s"), image, pOpts.Monitors, pOpts.Pool)
+	args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", cr.ID, "--keyfile=" + cr.KeyFile}
+	cmd, err := execCommand("rbd", args)
+	output = string(cmd)
+
+	if err, ok := err.(*exec.Error); ok {
+		if err.Err == exec.ErrNotFound {
+			klog.Errorf(util.Log(ctx, "rbd cmd not found"))
+			// fail fast if command not found
+			return false, output, err
+		}
+	}
+
+	// If command never succeed, returns its last error.
+	if err != nil {
+		return false, output, err
+	}
+
+	if strings.Contains(output, imageWatcherStr) {
+		klog.V(4).Infof(util.Log(ctx, "rbd: watchers on %s: %s"), image, output)
+		return true, output, nil
+	}
+	klog.Warningf(util.Log(ctx, "rbd: no watchers on %s"), image)
+	return false, output, nil
+}
+
+// rbdManagerTaskDelete adds a ceph manager task to delete an rbd image, thus deleting
+// it asynchronously. If command is not found returns a bool set to false
+func rbdManagerTaskDeleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) (bool, error) {
+	var output []byte
+
+	args := []string{"rbd", "task", "add", "remove",
+		pOpts.Pool + "/" + pOpts.RbdImageName,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-m", pOpts.Monitors,
+	}
+
+	output, err := execCommand("ceph", args)
+	if err != nil {
+		switch {
+		case strings.Contains(string(output), rbdTaskRemoveCmdInvalidString1) &&
+			strings.Contains(string(output), rbdTaskRemoveCmdInvalidString2):
+			klog.Warningf(util.Log(ctx, "cluster with cluster ID (%s) does not support Ceph manager based rbd image"+
+				" deletion (minimum ceph version required is v14.2.3)"), pOpts.ClusterID)
+		case strings.HasPrefix(string(output), rbdTaskRemoveCmdAccessDeniedMessage):
+			klog.Warningf(util.Log(ctx, "access denied to Ceph MGR-based RBD image deletion "+
+				"on cluster ID (%s)"), pOpts.ClusterID)
+		default:
+			klog.Warningf(util.Log(ctx, "uncaught error while scheduling an image deletion task: %s"), err)
+		}
+		return false, err
+	}
+
+	return true, err
+}
+
+// deleteImage deletes a ceph image with provision and volume options.
+func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) error {
+	var output []byte
+
+	image := pOpts.RbdImageName
+	found, _, err := rbdStatus(ctx, pOpts, cr)
+	if err != nil {
+		return err
+	}
+	if found {
+		klog.Errorf(util.Log(ctx, "rbd is still being used "), image)
+		return fmt.Errorf("rbd %s is still being used", image)
+	}
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: rm %s using mon %s, pool %s"), image, pOpts.Monitors, pOpts.Pool)
+
+	// attempt to use Ceph manager based deletion support if available
+	rbdCephMgrSupported, err := rbdManagerTaskDeleteImage(ctx, pOpts, cr)
+	if rbdCephMgrSupported && err != nil {
+		klog.Errorf(util.Log(ctx, "failed to add task to delete rbd image: %s/%s, %v"), pOpts.Pool, image, err)
+		return err
+	}
+
+	if !rbdCephMgrSupported {
+		// attempt older style deletion
+		args := []string{"rm", image, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors,
+			"--keyfile=" + cr.KeyFile}
+		output, err = execCommand("rbd", args)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s, error: %v, command output: %s"), pOpts.Pool, image, err, string(output))
+		}
+	}
+
+	return err
+}
+
+// updateSnapWithImageInfo updates provided rbdSnapshot with information from on-disk data
+// regarding the same
+func updateSnapWithImageInfo(ctx context.Context, rbdSnap *rbdSnapshot, cr *util.Credentials) error {
+	snapInfo, err := getSnapInfo(ctx, rbdSnap.Monitors, cr, rbdSnap.Pool,
+		rbdSnap.RbdImageName, rbdSnap.RbdSnapName)
+	if err != nil {
+		return err
+	}
+
+	rbdSnap.SizeBytes = snapInfo.Size
+
+	tm, err := time.Parse(time.ANSIC, snapInfo.Timestamp)
+	if err != nil {
+		return err
+	}
+
+	rbdSnap.CreatedAt, err = ptypes.TimestampProto(tm)
+
+	return err
+}
+
+// updateVolWithImageInfo updates provided rbdVolume with information from on-disk data
+// regarding the same
+func updateVolWithImageInfo(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
+	imageInfo, err := getImageInfo(ctx, rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RbdImageName)
+	if err != nil {
+		return err
+	}
+
+	rbdVol.VolSize = imageInfo.Size
+	rbdVol.ImageFeatures = strings.Join(imageInfo.Features, ",")
+
+	return nil
+}
+
+// genSnapFromSnapID generates a rbdSnapshot structure from the provided identifier, updating
+// the structure with elements from on-disk snapshot metadata as well
+func genSnapFromSnapID(ctx context.Context, rbdSnap *rbdSnapshot, snapshotID string, cr *util.Credentials) error {
+	var (
+		options map[string]string
+		vi      util.CSIIdentifier
+	)
+	options = make(map[string]string)
+
+	rbdSnap.SnapID = snapshotID
+
+	err := vi.DecomposeCSIID(rbdSnap.SnapID)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "error decoding snapshot ID (%s) (%s)"), err, rbdSnap.SnapID)
+		return err
+	}
+
+	rbdSnap.ClusterID = vi.ClusterID
+	options["clusterID"] = rbdSnap.ClusterID
+
+	rbdSnap.Monitors, _, err = getMonsAndClusterID(ctx, options)
+	if err != nil {
+		return err
+	}
+
+	rbdSnap.Pool, err = util.GetPoolName(ctx, rbdSnap.Monitors, cr, vi.LocationID)
+	if err != nil {
+		return err
+	}
+	rbdSnap.JournalPool = rbdSnap.Pool
+
+	imageAttributes, err := snapJournal.GetImageAttributes(ctx, rbdSnap.Monitors,
+		cr, rbdSnap.Pool, vi.ObjectUUID, true)
+	if err != nil {
+		return err
+	}
+	rbdSnap.RequestName = imageAttributes.RequestName
+	rbdSnap.RbdImageName = imageAttributes.SourceName
+	rbdSnap.RbdSnapName = imageAttributes.ImageName
+
+	// convert the journal pool ID to name, for use in DeleteSnapshot cases
+	if imageAttributes.JournalPoolID != util.InvalidPoolID {
+		rbdSnap.JournalPool, err = util.GetPoolName(ctx, rbdSnap.Monitors, cr, imageAttributes.JournalPoolID)
+		if err != nil {
+			// TODO: If pool is not found we may leak the image (as DeleteSnapshot will return success)
+			return err
+		}
+	}
+
+	err = updateSnapWithImageInfo(ctx, rbdSnap, cr)
+
+	return err
+}
+
+// genVolFromVolID generates a rbdVolume structure from the provided identifier, updating
+// the structure with elements from on-disk image metadata as well
+func genVolFromVolID(ctx context.Context, rbdVol *rbdVolume, volumeID string, cr *util.Credentials, secrets map[string]string) error {
+	var (
+		options map[string]string
+		vi      util.CSIIdentifier
+	)
+	options = make(map[string]string)
+
+	// rbdVolume fields that are not filled up in this function are:
+	//		Mounter, MultiNodeWritable
+	rbdVol.VolID = volumeID
+
+	err := vi.DecomposeCSIID(rbdVol.VolID)
+	if err != nil {
+		err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, rbdVol.VolID)
+		return ErrInvalidVolID{err}
+	}
+
+	rbdVol.ClusterID = vi.ClusterID
+	options["clusterID"] = rbdVol.ClusterID
+
+	rbdVol.Monitors, _, err = getMonsAndClusterID(ctx, options)
+	if err != nil {
+		return err
+	}
+
+	rbdVol.Pool, err = util.GetPoolName(ctx, rbdVol.Monitors, cr, vi.LocationID)
+	if err != nil {
+		return err
+	}
+	rbdVol.JournalPool = rbdVol.Pool
+
+	imageAttributes, err := volJournal.GetImageAttributes(ctx, rbdVol.Monitors, cr,
+		rbdVol.Pool, vi.ObjectUUID, false)
+	if err != nil {
+		return err
+	}
+
+	if imageAttributes.KmsID != "" {
+		rbdVol.Encrypted = true
+		rbdVol.KMS, err = util.GetKMS(imageAttributes.KmsID, secrets)
+		if err != nil {
+			return err
+		}
+	}
+	rbdVol.RequestName = imageAttributes.RequestName
+	rbdVol.RbdImageName = imageAttributes.ImageName
+
+	// convert the journal pool ID to name, for use in DeleteVolume cases
+	if imageAttributes.JournalPoolID >= 0 {
+		rbdVol.JournalPool, err = util.GetPoolName(ctx, rbdVol.Monitors, cr, imageAttributes.JournalPoolID)
+		if err != nil {
+			// TODO: If pool is not found we may leak the image (as DeleteVolume will return success)
+			return err
+		}
+	}
+
+	err = updateVolWithImageInfo(ctx, rbdVol, cr)
+
+	return err
+}
+
+func execCommand(command string, args []string) ([]byte, error) {
+	// #nosec
+	cmd := exec.Command(command, args...)
+	return cmd.CombinedOutput()
+}
+
+func getMonsAndClusterID(ctx context.Context, options map[string]string) (monitors, clusterID string, err error) {
+	var ok bool
+
+	if clusterID, ok = options["clusterID"]; !ok {
+		err = errors.New("clusterID must be set")
+		return
+	}
+
+	if monitors, err = util.Mons(csiConfigFile, clusterID); err != nil {
+		klog.Errorf(util.Log(ctx, "failed getting mons (%s)"), err)
+		err = errors.Wrapf(err, "failed to fetch monitor list using clusterID (%s)", clusterID)
+		return
+	}
+
+	return
+}
+
+// isLegacyVolumeID checks if passed in volume ID string conforms to volume ID naming scheme used
+// by the version 1.0.0 (legacy) of the plugin, and returns true if found to be conforming
+func isLegacyVolumeID(volumeID string) bool {
+	// Version 1.0.0 volumeID format: "csi-rbd-vol-" + UUID string
+	//    length: 12 ("csi-rbd-vol-") + 36 (UUID string)
+
+	// length check
+	if len(volumeID) != 48 {
+		return false
+	}
+
+	// Header check
+	if !strings.HasPrefix(volumeID, "csi-rbd-vol-") {
+		return false
+	}
+
+	// Trailer UUID format check
+	if uuid.Parse(volumeID[12:]) == nil {
+		return false
+	}
+
+	return true
+}
+
+// upadateMons function is used to update the rbdVolume.Monitors for volumes that were provisioned
+// using the 1.0.0 version (legacy) of the plugin.
+func updateMons(rbdVol *rbdVolume, options, credentials map[string]string) error {
+	var ok bool
+
+	// read monitors and MonValueFromSecret from options, else check passed in rbdVolume for
+	// MonValueFromSecret key in credentials
+	monInSecret := ""
+	if options != nil {
+		if rbdVol.Monitors, ok = options["monitors"]; !ok {
+			rbdVol.Monitors = ""
+		}
+		if monInSecret, ok = options["monValueFromSecret"]; !ok {
+			monInSecret = ""
+		}
+	} else {
+		monInSecret = rbdVol.MonValueFromSecret
+	}
+
+	// if monitors are present in secrets and we have the credentials, use monitors from the
+	// credentials overriding monitors from other sources
+	if monInSecret != "" && credentials != nil {
+		monsFromSecret, ok := credentials[monInSecret]
+		if ok {
+			rbdVol.Monitors = monsFromSecret
+		}
+	}
+
+	if rbdVol.Monitors == "" {
+		return errors.New("either monitors or monValueFromSecret must be set")
+	}
+
+	return nil
+}
+
+func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[string]string, disableInUseChecks, isLegacyVolume bool) (*rbdVolume, error) {
+	var (
+		ok         bool
+		err        error
+		namePrefix string
+		encrypted  string
+	)
+
+	rbdVol := &rbdVolume{}
+	rbdVol.Pool, ok = volOptions["pool"]
+	if !ok {
+		return nil, errors.New("missing required parameter pool")
+	}
+
+	rbdVol.DataPool = volOptions["dataPool"]
+	if namePrefix, ok = volOptions["volumeNamePrefix"]; ok {
+		rbdVol.NamePrefix = namePrefix
+	}
+
+	if isLegacyVolume {
+		err = updateMons(rbdVol, volOptions, credentials)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		rbdVol.Monitors, rbdVol.ClusterID, err = getMonsAndClusterID(ctx, volOptions)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// if no image features is provided, it results in empty string
+	// which disable all RBD image features as we expected
+
+	imageFeatures, found := volOptions["imageFeatures"]
+	if found {
+		arr := strings.Split(imageFeatures, ",")
+		for _, f := range arr {
+			if !supportedFeatures.Has(f) {
+				return nil, fmt.Errorf("invalid feature %q for volume csi-rbdplugin, supported"+
+					" features are: %v", f, supportedFeatures)
+			}
+		}
+		rbdVol.ImageFeatures = imageFeatures
+	}
+
+	klog.V(3).Infof(util.Log(ctx, "setting disableInUseChecks on rbd volume to: %v"), disableInUseChecks)
+	rbdVol.DisableInUseChecks = disableInUseChecks
+
+	rbdVol.Mounter, ok = volOptions["mounter"]
+	if !ok {
+		rbdVol.Mounter = rbdDefaultMounter
+	}
+
+	rbdVol.Encrypted = false
+	encrypted, ok = volOptions["encrypted"]
+	if ok {
+		rbdVol.Encrypted, err = strconv.ParseBool(encrypted)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"invalid value set in 'encrypted': %s (should be \"true\" or \"false\")", encrypted)
+		}
+
+		if rbdVol.Encrypted {
+			// deliberately ignore if parsing failed as GetKMS will return default
+			// implementation of kmsID is empty
+			kmsID := volOptions["encryptionKMSID"]
+			rbdVol.KMS, err = util.GetKMS(kmsID, credentials)
+			if err != nil {
+				return nil, fmt.Errorf("invalid encryption kms configuration: %s", err)
+			}
+		}
+	}
+
+	return rbdVol, nil
+}
+
+func genSnapFromOptions(ctx context.Context, rbdVol *rbdVolume, snapOptions map[string]string) *rbdSnapshot {
+	var err error
+
+	rbdSnap := &rbdSnapshot{}
+	rbdSnap.Pool = rbdVol.Pool
+	rbdSnap.JournalPool = rbdVol.JournalPool
+
+	rbdSnap.Monitors, rbdSnap.ClusterID, err = getMonsAndClusterID(ctx, snapOptions)
+	if err != nil {
+		rbdSnap.Monitors = rbdVol.Monitors
+		rbdSnap.ClusterID = rbdVol.ClusterID
+	}
+
+	if namePrefix, ok := snapOptions["snapshotNamePrefix"]; ok {
+		rbdSnap.NamePrefix = namePrefix
+	}
+
+	return rbdSnap
+}
+
+func hasSnapshotFeature(imageFeatures string) bool {
+	arr := strings.Split(imageFeatures, ",")
+	for _, f := range arr {
+		if f == "layering" {
+			return true
+		}
+	}
+	return false
+}
+
+// imageFeaturesToUint64 takes the comma separated image features and converts
+// them to a RbdImageOptionFeatures value.
+func imageFeaturesToUint64(ctx context.Context, imageFeatures string) uint64 {
+	features := uint64(0)
+
+	for _, f := range strings.Split(imageFeatures, ",") {
+		if f == "layering" {
+			features |= librbd.RbdFeatureLayering
+		} else {
+			klog.Warningf(util.Log(ctx, "rbd: image feature %s not recognized, skipping"), f)
+		}
+	}
+	return features
+}
+
+func protectSnapshot(ctx context.Context, pOpts *rbdSnapshot, cr *util.Credentials) error {
+	var output []byte
+
+	image := pOpts.RbdImageName
+	snapName := pOpts.RbdSnapName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: snap protect %s using mon %s, pool %s "), image, pOpts.Monitors, pOpts.Pool)
+	args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
+		cr.ID, "-m", pOpts.Monitors, "--keyfile=" + cr.KeyFile}
+
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to protect snapshot, command output: %s", string(output))
+	}
+
+	return nil
+}
+
+func createSnapshot(ctx context.Context, pOpts *rbdSnapshot, cr *util.Credentials) error {
+	var output []byte
+
+	image := pOpts.RbdImageName
+	snapName := pOpts.RbdSnapName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: snap create %s using mon %s, pool %s"), image, pOpts.Monitors, pOpts.Pool)
+	args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapName, image,
+		"--id", cr.ID, "-m", pOpts.Monitors, "--keyfile=" + cr.KeyFile}
+
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to create snapshot, command output: %s", string(output))
+	}
+
+	return nil
+}
+
+func unprotectSnapshot(ctx context.Context, pOpts *rbdSnapshot, cr *util.Credentials) error {
+	var output []byte
+
+	image := pOpts.RbdImageName
+	snapName := pOpts.RbdSnapName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: snap unprotect %s using mon %s, pool %s"), image, pOpts.Monitors, pOpts.Pool)
+	args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
+		cr.ID, "-m", pOpts.Monitors, "--keyfile=" + cr.KeyFile}
+
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to unprotect snapshot, command output: %s", string(output))
+	}
+
+	return nil
+}
+
+func deleteSnapshot(ctx context.Context, pOpts *rbdSnapshot, cr *util.Credentials) error {
+	var output []byte
+
+	image := pOpts.RbdImageName
+	snapName := pOpts.RbdSnapName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: snap rm %s using mon %s, pool %s"), image, pOpts.Monitors, pOpts.Pool)
+	args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
+		cr.ID, "-m", pOpts.Monitors, "--keyfile=" + cr.KeyFile}
+
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to delete snapshot, command output: %s", string(output))
+	}
+
+	if err := undoSnapReservation(ctx, pOpts, cr); err != nil {
+		klog.Errorf(util.Log(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)"),
+			pOpts.RequestName, pOpts.RbdSnapName, pOpts.RbdImageName, err)
+	}
+
+	return nil
+}
+
+func restoreSnapshot(ctx context.Context, pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, cr *util.Credentials) error {
+	var output []byte
+
+	image := pVolOpts.RbdImageName
+	snapName := pSnapOpts.RbdSnapName
+
+	klog.V(4).Infof(util.Log(ctx, "rbd: clone %s using mon %s, pool %s"), image, pVolOpts.Monitors, pVolOpts.Pool)
+	args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.RbdImageName + "@" + snapName,
+		pVolOpts.Pool + "/" + image, "--id", cr.ID, "-m", pVolOpts.Monitors, "--keyfile=" + cr.KeyFile}
+
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to restore snapshot, command output: %s", string(output))
+	}
+
+	return nil
+}
+
+// getSnapshotMetadata fetches on-disk metadata about the snapshot and populates the passed in
+// rbdSnapshot structure
+func getSnapshotMetadata(ctx context.Context, pSnapOpts *rbdSnapshot, cr *util.Credentials) error {
+	imageName := pSnapOpts.RbdImageName
+	snapName := pSnapOpts.RbdSnapName
+
+	snapInfo, err := getSnapInfo(ctx, pSnapOpts.Monitors, cr, pSnapOpts.Pool, imageName, snapName)
+	if err != nil {
+		return err
+	}
+
+	pSnapOpts.SizeBytes = snapInfo.Size
+
+	tm, err := time.Parse(time.ANSIC, snapInfo.Timestamp)
+	if err != nil {
+		return err
+	}
+
+	pSnapOpts.CreatedAt, err = ptypes.TimestampProto(tm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// imageInfo strongly typed JSON spec for image info
+type imageInfo struct {
+	ObjectUUID string   `json:"name"`
+	Size       int64    `json:"size"`
+	Features   []string `json:"features"`
+	CreatedAt  string   `json:"create_timestamp"`
+}
+
+// getImageInfo queries rbd about the given image and returns its metadata, and returns
+// ErrImageNotFound if provided image is not found
+func getImageInfo(ctx context.Context, monitors string, cr *util.Credentials, poolName, imageName string) (imageInfo, error) {
+	// rbd --format=json info [image-spec | snap-spec]
+
+	var imgInfo imageInfo
+
+	stdout, stderr, err := util.ExecCommand(
+		"rbd",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", util.CephConfigPath,
+		"--format="+"json",
+		"info", poolName+"/"+imageName)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed getting information for image (%s): (%s)"), poolName+"/"+imageName, err)
+		if strings.Contains(string(stderr), "rbd: error opening image "+imageName+
+			": (2) No such file or directory") {
+			return imgInfo, ErrImageNotFound{imageName, err}
+		}
+		return imgInfo, err
+	}
+
+	err = json.Unmarshal(stdout, &imgInfo)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to parse JSON output of image info (%s): (%s)"),
+			poolName+"/"+imageName, err)
+		return imgInfo, fmt.Errorf("unmarshal failed: %+v.  raw buffer response: %s",
+			err, string(stdout))
+	}
+
+	return imgInfo, nil
+}
+
+// snapInfo strongly typed JSON spec for snap ls rbd output
+type snapInfo struct {
+	ID        int64  `json:"id"`
+	Name      string `json:"name"`
+	Size      int64  `json:"size"`
+	Timestamp string `json:"timestamp"`
+}
+
+/*
+getSnapInfo queries rbd about the snapshots of the given image and returns its metadata, and
+returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if provided snap
+is not found in the images snapshot list
+*/
+func getSnapInfo(ctx context.Context, monitors string, cr *util.Credentials, poolName, imageName, snapName string) (snapInfo, error) {
+	// rbd --format=json snap ls [image-spec]
+
+	var (
+		snpInfo snapInfo
+		snaps   []snapInfo
+	)
+
+	stdout, stderr, err := util.ExecCommand(
+		"rbd",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", util.CephConfigPath,
+		"--format="+"json",
+		"snap", "ls", poolName+"/"+imageName)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed getting snap (%s) information from image (%s): (%s)"),
+			snapName, poolName+"/"+imageName, err)
+		if strings.Contains(string(stderr), "rbd: error opening image "+imageName+
+			": (2) No such file or directory") {
+			return snpInfo, ErrImageNotFound{imageName, err}
+		}
+		return snpInfo, err
+	}
+
+	err = json.Unmarshal(stdout, &snaps)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to parse JSON output of image snap list (%s): (%s)"),
+			poolName+"/"+imageName, err)
+		return snpInfo, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s",
+			err, string(stdout))
+	}
+
+	for _, snap := range snaps {
+		if snap.Name == snapName {
+			return snap, nil
+		}
+	}
+
+	return snpInfo, ErrSnapNotFound{snapName, fmt.Errorf("snap (%s) for image (%s) not found",
+		snapName, poolName+"/"+imageName)}
+}
+
+// rbdImageMetadataStash strongly typed JSON spec for stashed RBD image metadata
+type rbdImageMetadataStash struct {
+	Version   int    `json:"Version"`
+	Pool      string `json:"pool"`
+	ImageName string `json:"image"`
+	NbdAccess bool   `json:"accessType"`
+	Encrypted bool   `json:"encrypted"`
+}
+
+// file name in which image metadata is stashed
+const stashFileName = "image-meta.json"
+
+// stashRBDImageMetadata stashes required fields into the stashFileName at the passed in path, in
+// JSON format
+func stashRBDImageMetadata(volOptions *rbdVolume, path string) error {
+	var imgMeta = rbdImageMetadataStash{
+		Version:   2, // there are no checks for this at present
+		Pool:      volOptions.Pool,
+		ImageName: volOptions.RbdImageName,
+		Encrypted: volOptions.Encrypted,
+	}
+
+	imgMeta.NbdAccess = false
+	if volOptions.Mounter == rbdTonbd && hasNBD {
+		imgMeta.NbdAccess = true
+	}
+
+	encodedBytes, err := json.Marshal(imgMeta)
+	if err != nil {
+		return fmt.Errorf("failed to marshall JSON image metadata for image (%s) in pool (%s): (%v)",
+			volOptions.RbdImageName, volOptions.Pool, err)
+	}
+
+	fPath := filepath.Join(path, stashFileName)
+	err = ioutil.WriteFile(fPath, encodedBytes, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to stash JSON image metadata for image (%s) in pool (%s) at path (%s): (%v)",
+			volOptions.RbdImageName, volOptions.Pool, fPath, err)
+	}
+
+	return nil
+}
+
+// lookupRBDImageMetadataStash reads and returns stashed image metadata at passed in path
+func lookupRBDImageMetadataStash(path string) (rbdImageMetadataStash, error) {
+	var imgMeta rbdImageMetadataStash
+
+	fPath := filepath.Join(path, stashFileName)
+	encodedBytes, err := ioutil.ReadFile(fPath) // #nosec - intended reading from fPath
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return imgMeta, fmt.Errorf("failed to read stashed JSON image metadata from path (%s): (%v)", fPath, err)
+		}
+
+		return imgMeta, ErrMissingStash{err}
+	}
+
+	err = json.Unmarshal(encodedBytes, &imgMeta)
+	if err != nil {
+		return imgMeta, fmt.Errorf("failed to unmarshall stashed JSON image metadata from path (%s): (%v)", fPath, err)
+	}
+
+	return imgMeta, nil
+}
+
+// cleanupRBDImageMetadataStash cleans up any stashed metadata at passed in path
+func cleanupRBDImageMetadataStash(path string) error {
+	fPath := filepath.Join(path, stashFileName)
+	if err := os.Remove(fPath); err != nil {
+		return fmt.Errorf("failed to cleanup stashed JSON data (%s): (%v)", fPath, err)
+	}
+
+	return nil
+}
+
+// resizeRBDImage resizes the given volume to new size
+func resizeRBDImage(rbdVol *rbdVolume, cr *util.Credentials) error {
+	var output []byte
+
+	mon := rbdVol.Monitors
+	image := rbdVol.RbdImageName
+	volSzMiB := fmt.Sprintf("%dM", util.RoundOffVolSize(rbdVol.VolSize))
+
+	args := []string{"resize", image, "--size", volSzMiB, "--pool", rbdVol.Pool, "--id", cr.ID, "-m", mon, "--keyfile=" + cr.KeyFile}
+	output, err := execCommand("rbd", args)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to resize rbd image, command output: %s", string(output))
+	}
+
+	return nil
+}
+
+func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error {
+	var vi util.CSIIdentifier
+
+	err := vi.DecomposeCSIID(rbdVol.VolID)
+	if err != nil {
+		err = fmt.Errorf("error decoding volume ID (%s) (%s)", rbdVol.VolID, err)
+		return ErrInvalidVolID{err}
+	}
+
+	rbdImageName := volJournal.GetNameForUUID(rbdVol.NamePrefix, vi.ObjectUUID, false)
+	imageSpec := rbdVol.Pool + "/" + rbdImageName
+
+	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption)
+	if err != nil {
+		return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err)
+	}
+
+	return nil
+}

internal/rbd/rbd_util_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"testing"
+)
+
+func TestIsLegacyVolumeID(t *testing.T) {
+	tests := []struct {
+		volID    string
+		isLegacy bool
+	}{
+		{"prefix-bda37d42-9979-420f-9389-74362f3f98f6", false},
+		{"csi-rbd-vo-f997e783-ff00-48b0-8cc7-30cb36c3df3d", false},
+		{"csi-rbd-vol-this-is-clearly-not-a-valid-UUID----", false},
+		{"csi-rbd-vol-b82f27de-3b3a-43f2-b5e7-9f8d0aad04e9", true},
+	}
+
+	for _, test := range tests {
+		if got := isLegacyVolumeID(test.volID); got != test.isLegacy {
+			t.Errorf("isLegacyVolumeID(%s) = %t, want %t", test.volID, got, test.isLegacy)
+		}
+	}
+}
+
+func TestHasSnapshotFeature(t *testing.T) {
+	tests := []struct {
+		features   string
+		hasFeature bool
+	}{
+		{"foo", false},
+		{"foo,bar", false},
+		{"foo,layering,bar", true},
+	}
+
+	for _, test := range tests {
+		if got := hasSnapshotFeature(test.features); got != test.hasFeature {
+			t.Errorf("hasSnapshotFeature(%s) = %t, want %t", test.features, got, test.hasFeature)
+		}
+	}
+}

internal/util/cachepersister.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"errors"
+
+	"k8s.io/klog"
+)
+
+// ForAllFunc is a unary predicate for visiting all cache entries
+// matching the `pattern' in CachePersister's ForAll function.
+type ForAllFunc func(identifier string) error
+
+// CacheEntryNotFound is an error type for "Not Found" cache errors
+type CacheEntryNotFound struct {
+	error
+}
+
+// CachePersister interface implemented for store
+type CachePersister interface {
+	Create(identifier string, data interface{}) error
+	Get(identifier string, data interface{}) error
+	ForAll(pattern string, destObj interface{}, f ForAllFunc) error
+	Delete(identifier string) error
+}
+
+// NewCachePersister returns CachePersister based on store
+func NewCachePersister(metadataStore, pluginPath string) (CachePersister, error) {
+	if metadataStore == "k8s_configmap" {
+		klog.V(4).Infof("cache-perister: using kubernetes configmap as metadata cache persister")
+		k8scm := &K8sCMCache{}
+		k8scm.Client = NewK8sClient()
+		k8scm.Namespace = GetK8sNamespace()
+		return k8scm, nil
+	} else if metadataStore == "node" {
+		klog.V(4).Infof("cache-persister: using node as metadata cache persister")
+		nc := &NodeCache{}
+		nc.BasePath = pluginPath
+		nc.CacheDir = "controller"
+		return nc, nil
+	}
+	return nil, errors.New("cache-persister: couldn't parse metadatastorage flag")
+}

internal/util/cephcmds.go

@@ -0,0 +1,355 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+
+	"k8s.io/klog"
+)
+
+// InvalidPoolID used to denote an invalid pool
+const InvalidPoolID int64 = -1
+
+// ExecCommand executes passed in program with args and returns separate stdout and stderr streams
+func ExecCommand(program string, args ...string) (stdout, stderr []byte, err error) {
+	var (
+		cmd           = exec.Command(program, args...) // nolint: gosec, #nosec
+		sanitizedArgs = StripSecretInArgs(args)
+		stdoutBuf     bytes.Buffer
+		stderrBuf     bytes.Buffer
+	)
+
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = &stderrBuf
+
+	if err := cmd.Run(); err != nil {
+		return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+
+			" occurred while running %s args: %v", err, program, sanitizedArgs)
+	}
+
+	return stdoutBuf.Bytes(), nil, nil
+}
+
+// cephStoragePoolSummary strongly typed JSON spec for osd ls pools output
+type cephStoragePoolSummary struct {
+	Name   string `json:"poolname"`
+	Number int64  `json:"poolnum"`
+}
+
+// GetPools fetches a list of pools from a cluster
+func getPools(ctx context.Context, monitors string, cr *Credentials) ([]cephStoragePoolSummary, error) {
+	// ceph <options> -f json osd lspools
+	// JSON out: [{"poolnum":<int64>,"poolname":<string>}]
+
+	stdout, _, err := ExecCommand(
+		"ceph",
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile="+cr.KeyFile,
+		"-c", CephConfigPath,
+		"-f", "json",
+		"osd", "lspools")
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed getting pool list from cluster (%s)"), err)
+		return nil, err
+	}
+
+	var pools []cephStoragePoolSummary
+	err = json.Unmarshal(stdout, &pools)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed to parse JSON output of pool list from cluster (%s)"), err)
+		return nil, fmt.Errorf("unmarshal of pool list failed: %+v.  raw buffer response: %s", err, string(stdout))
+	}
+
+	return pools, nil
+}
+
+// GetPoolID searches a list of pools in a cluster and returns the ID of the pool that matches
+// the passed in poolName parameter
+func GetPoolID(ctx context.Context, monitors string, cr *Credentials, poolName string) (int64, error) {
+	pools, err := getPools(ctx, monitors, cr)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, p := range pools {
+		if poolName == p.Name {
+			return p.Number, nil
+		}
+	}
+
+	return 0, fmt.Errorf("pool (%s) not found in Ceph cluster", poolName)
+}
+
+// GetPoolName lists all pools in a ceph cluster, and matches the pool whose pool ID is equal to
+// the requested poolID parameter
+func GetPoolName(ctx context.Context, monitors string, cr *Credentials, poolID int64) (string, error) {
+	pools, err := getPools(ctx, monitors, cr)
+	if err != nil {
+		return "", err
+	}
+
+	for _, p := range pools {
+		if poolID == p.Number {
+			return p.Name, nil
+		}
+	}
+
+	return "", ErrPoolNotFound{string(poolID), fmt.Errorf("pool ID (%d) not found in Ceph cluster", poolID)}
+}
+
+// GetPoolIDs searches a list of pools in a cluster and returns the IDs of the pools that matches
+// the passed in pools
+// TODO this should take in a list and return a map[string(poolname)]int64(poolID)
+func GetPoolIDs(ctx context.Context, monitors, journalPool, imagePool string, cr *Credentials) (int64, int64, error) {
+	journalPoolID, err := GetPoolID(ctx, monitors, cr, journalPool)
+	if err != nil {
+		return InvalidPoolID, InvalidPoolID, err
+	}
+
+	imagePoolID := journalPoolID
+	if imagePool != journalPool {
+		imagePoolID, err = GetPoolID(ctx, monitors, cr, imagePool)
+		if err != nil {
+			return InvalidPoolID, InvalidPoolID, err
+		}
+	}
+
+	return journalPoolID, imagePoolID, nil
+}
+
+// SetOMapKeyValue sets the given key and value into the provided Ceph omap name
+func SetOMapKeyValue(ctx context.Context, monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey, keyValue string) error {
+	// Command: "rados <options> setomapval oMapName oMapKey keyValue"
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"-p", poolName,
+		"setomapval", oMapName, oMapKey, keyValue,
+	}
+
+	if namespace != "" {
+		args = append(args, "--namespace="+namespace)
+	}
+
+	_, _, err := ExecCommand("rados", args[:]...)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed adding key (%s with value %s), to omap (%s) in "+
+			"pool (%s): (%v)"), oMapKey, keyValue, oMapName, poolName, err)
+		return err
+	}
+
+	return nil
+}
+
+// GetOMapValue gets the value for the given key from the named omap
+func GetOMapValue(ctx context.Context, monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) (string, error) {
+	// Command: "rados <options> getomapval oMapName oMapKey <outfile>"
+	// No such key: replicapool/csi.volumes.directory.default/csi.volname
+	tmpFile, err := ioutil.TempFile("", "omap-get-")
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed creating a temporary file for key contents"))
+		return "", err
+	}
+	defer tmpFile.Close()
+	defer os.Remove(tmpFile.Name())
+
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"-p", poolName,
+		"getomapval", oMapName, oMapKey, tmpFile.Name(),
+	}
+
+	if namespace != "" {
+		args = append(args, "--namespace="+namespace)
+	}
+
+	stdout, stderr, err := ExecCommand("rados", args[:]...)
+	if err != nil {
+		// no logs, as attempting to check for non-existent key/value is done even on
+		// regular call sequences
+		stdoutanderr := strings.Join([]string{string(stdout), string(stderr)}, " ")
+		if strings.Contains(stdoutanderr, "No such key: "+poolName+"/"+oMapName+"/"+oMapKey) {
+			return "", ErrKeyNotFound{poolName + "/" + oMapName + "/" + oMapKey, err}
+		}
+
+		if strings.Contains(stdoutanderr, "error getting omap value "+
+			poolName+"/"+oMapName+"/"+oMapKey+": (2) No such file or directory") {
+			return "", ErrKeyNotFound{poolName + "/" + oMapName + "/" + oMapKey, err}
+		}
+
+		if strings.Contains(stdoutanderr, "error opening pool "+
+			poolName+": (2) No such file or directory") {
+			return "", ErrPoolNotFound{poolName, err}
+		}
+
+		// log other errors for troubleshooting assistance
+		klog.Errorf(Log(ctx, "failed getting omap value for key (%s) from omap (%s) in pool (%s): (%v)"),
+			oMapKey, oMapName, poolName, err)
+
+		return "", fmt.Errorf("error (%v) occurred, command output streams is (%s)",
+			err.Error(), stdoutanderr)
+	}
+
+	keyValue, err := ioutil.ReadAll(tmpFile)
+	return string(keyValue), err
+}
+
+// RemoveOMapKey removes the omap key from the given omap name
+func RemoveOMapKey(ctx context.Context, monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) error {
+	// Command: "rados <options> rmomapkey oMapName oMapKey"
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"-p", poolName,
+		"rmomapkey", oMapName, oMapKey,
+	}
+
+	if namespace != "" {
+		args = append(args, "--namespace="+namespace)
+	}
+
+	_, _, err := ExecCommand("rados", args[:]...)
+	if err != nil {
+		// NOTE: Missing omap key removal does not return an error
+		klog.Errorf(Log(ctx, "failed removing key (%s), from omap (%s) in "+
+			"pool (%s): (%v)"), oMapKey, oMapName, poolName, err)
+		return err
+	}
+
+	return nil
+}
+
+// CreateObject creates the object name passed in and returns ErrObjectExists if the provided object
+// is already present in rados
+func CreateObject(ctx context.Context, monitors string, cr *Credentials, poolName, namespace, objectName string) error {
+	// Command: "rados <options> create objectName"
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"-p", poolName,
+		"create", objectName,
+	}
+
+	if namespace != "" {
+		args = append(args, "--namespace="+namespace)
+	}
+
+	_, stderr, err := ExecCommand("rados", args[:]...)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed creating omap (%s) in pool (%s): (%v)"), objectName, poolName, err)
+		if strings.Contains(string(stderr), "error creating "+poolName+"/"+objectName+
+			": (17) File exists") {
+			return ErrObjectExists{objectName, err}
+		}
+		return err
+	}
+
+	return nil
+}
+
+// RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap
+// is not found in rados
+func RemoveObject(ctx context.Context, monitors string, cr *Credentials, poolName, namespace, oMapName string) error {
+	// Command: "rados <options> rm oMapName"
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"-p", poolName,
+		"rm", oMapName,
+	}
+
+	if namespace != "" {
+		args = append(args, "--namespace="+namespace)
+	}
+
+	_, stderr, err := ExecCommand("rados", args[:]...)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed removing omap (%s) in pool (%s): (%v)"), oMapName, poolName, err)
+		if strings.Contains(string(stderr), "error removing "+poolName+">"+oMapName+
+			": (2) No such file or directory") {
+			return ErrObjectNotFound{oMapName, err}
+		}
+		return err
+	}
+
+	return nil
+}
+
+// SetImageMeta sets image metadata
+func SetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key, value string) error {
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"image-meta", "set", imageSpec,
+		key, value,
+	}
+
+	_, _, err := ExecCommand("rbd", args[:]...)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed setting image metadata (%s) for (%s): (%v)"), key, imageSpec, err)
+		return err
+	}
+
+	return nil
+}
+
+// GetImageMeta gets image metadata
+func GetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key string) (string, error) {
+	args := []string{
+		"-m", monitors,
+		"--id", cr.ID,
+		"--keyfile=" + cr.KeyFile,
+		"-c", CephConfigPath,
+		"image-meta", "get", imageSpec,
+		key,
+	}
+
+	stdout, stderr, err := ExecCommand("rbd", args[:]...)
+	if err != nil {
+		stdoutanderr := strings.Join([]string{string(stdout), string(stderr)}, " ")
+		if strings.Contains(stdoutanderr, "failed to get metadata "+key+" of image : (2) No such file or directory") {
+			return "", ErrKeyNotFound{imageSpec + " " + key, err}
+		}
+
+		klog.Errorf(Log(ctx, "failed getting image metadata (%s) for (%s): (%v)"), key, imageSpec, err)
+		return "", err
+	}
+
+	return string(stdout), nil
+}

internal/util/cephconf.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+var cephConfig = []byte(`[global]
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+# Workaround for http://tracker.ceph.com/issues/23446
+fuse_set_user_groups = false
+`)
+
+const (
+	cephConfigRoot = "/etc/ceph"
+	// CephConfigPath ceph configuration file
+	CephConfigPath = "/etc/ceph/ceph.conf"
+
+	keyRing = "/etc/ceph/keyring"
+)
+
+func createCephConfigRoot() error {
+	return os.MkdirAll(cephConfigRoot, 0755) // #nosec
+}
+
+// WriteCephConfig writes out a basic ceph.conf file, making it easy to use
+// ceph related CLIs
+func WriteCephConfig() error {
+	if err := createCephConfigRoot(); err != nil {
+		return err
+	}
+
+	err := ioutil.WriteFile(CephConfigPath, cephConfig, 0640)
+	if err != nil {
+		return err
+	}
+
+	return createKeyRingFile()
+}
+
+/*
+if any ceph commands fails it will log below error message
+
+7f39ff02a700 -1 auth: unable to find a keyring on
+/etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,
+/etc/ceph/keyring.bin,: (2) No such file or directory
+*/
+// createKeyRingFile creates the keyring files to fix above error message logging
+func createKeyRingFile() error {
+	_, err := os.Create(keyRing)
+	return err
+}

internal/util/conn_pool.go

@@ -0,0 +1,206 @@
+/*
+Copyright 2020 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sync"
+	"time"
+
+	"github.com/ceph/go-ceph/rados"
+	"github.com/pkg/errors"
+)
+
+type connEntry struct {
+	conn     *rados.Conn
+	lastUsed time.Time
+	users    int
+}
+
+// ConnPool is the struct which contains details of connection entries in the pool and gc controlled params.
+type ConnPool struct {
+	// interval to run the garbage collector
+	interval time.Duration
+	// timeout for a connEntry to get garbage collected
+	expiry time.Duration
+	// Timer used to schedule calls to the garbage collector
+	timer *time.Timer
+	// Mutex for loading and touching connEntry's from the conns Map
+	lock *sync.RWMutex
+	// all connEntry's in this pool
+	conns map[string]*connEntry
+}
+
+// NewConnPool creates a new connection pool instance and start the garbage collector running
+// every @interval.
+func NewConnPool(interval, expiry time.Duration) *ConnPool {
+	cp := ConnPool{
+		interval: interval,
+		expiry:   expiry,
+		lock:     &sync.RWMutex{},
+		conns:    make(map[string]*connEntry),
+	}
+	cp.timer = time.AfterFunc(interval, cp.gc)
+
+	return &cp
+}
+
+// loop through all cp.conns and destroy objects that have not been used for cp.expiry.
+func (cp *ConnPool) gc() {
+	cp.lock.Lock()
+	defer cp.lock.Unlock()
+
+	now := time.Now()
+	for key, ce := range cp.conns {
+		if ce.users == 0 && (now.Sub(ce.lastUsed)) > cp.expiry {
+			ce.destroy()
+			delete(cp.conns, key)
+		}
+	}
+
+	// schedule the next gc() run
+	cp.timer.Reset(cp.interval)
+}
+
+// Destroy stops the garbage collector and destroys all connections in the pool.
+func (cp *ConnPool) Destroy() {
+	cp.timer.Stop()
+	// wait until gc() has finished, in case it is running
+	cp.lock.Lock()
+	defer cp.lock.Unlock()
+
+	for key, ce := range cp.conns {
+		if ce.users != 0 {
+			panic("this connEntry still has users, operations" +
+				"might still be in-flight")
+		}
+
+		ce.destroy()
+		delete(cp.conns, key)
+	}
+}
+
+func (cp *ConnPool) generateUniqueKey(pool, monitors, user, keyfile string) (string, error) {
+	// the keyfile can be unique for operations, contents will be the same
+	key, err := ioutil.ReadFile(keyfile) // nolint: gosec, #nosec
+	if err != nil {
+		return "", errors.Wrapf(err, "could not open keyfile %s", keyfile)
+	}
+
+	return fmt.Sprintf("%s|%s|%s|%s", pool, monitors, user, string(key)), nil
+}
+
+// getExisting returns the existing rados.Conn associated with the unique key.
+//
+// Requires: locked cp.lock because of ce.get()
+func (cp *ConnPool) getConn(unique string) *rados.Conn {
+	ce, exists := cp.conns[unique]
+	if exists {
+		ce.get()
+		return ce.conn
+	}
+	return nil
+}
+
+// Get returns a rados.Conn for the given arguments. Creates a new rados.Conn in
+// case there is none in the pool. Use the returned unique string to reduce the
+// reference count with ConnPool.Put(unique).
+func (cp *ConnPool) Get(pool, monitors, user, keyfile string) (*rados.Conn, error) {
+	unique, err := cp.generateUniqueKey(pool, monitors, user, keyfile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to generate unique for connection")
+	}
+
+	cp.lock.RLock()
+	conn := cp.getConn(unique)
+	cp.lock.RUnlock()
+	if conn != nil {
+		return conn, nil
+	}
+
+	// construct and connect a new rados.Conn
+	args := []string{"-m", monitors, "--keyfile=" + keyfile}
+	conn, err = rados.NewConnWithUser(user)
+	if err != nil {
+		return nil, errors.Wrapf(err, "creating a new connection failed")
+	}
+	err = conn.ParseCmdLineArgs(args)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parsing cmdline args (%v) failed", args)
+	}
+
+	err = conn.Connect()
+	if err != nil {
+		return nil, errors.Wrapf(err, "connecting failed")
+	}
+
+	ce := &connEntry{
+		conn:     conn,
+		lastUsed: time.Now(),
+		users:    1,
+	}
+
+	cp.lock.Lock()
+	defer cp.lock.Unlock()
+	oldConn := cp.getConn(unique)
+	if oldConn != nil {
+		// there was a race, oldConn already exists
+		ce.destroy()
+		return oldConn, nil
+	}
+	// this really is a new connection, add it to the map
+	cp.conns[unique] = ce
+
+	return conn, nil
+}
+
+// Put reduces the reference count of the rados.Conn object that was returned with
+// ConnPool.Get().
+func (cp *ConnPool) Put(conn *rados.Conn) {
+	cp.lock.Lock()
+	defer cp.lock.Unlock()
+
+	for _, ce := range cp.conns {
+		if ce.conn == conn {
+			ce.put()
+			return
+		}
+	}
+}
+
+// Add a reference to the connEntry.
+// /!\ Only call this while holding the ConnPool.lock.
+func (ce *connEntry) get() {
+	ce.lastUsed = time.Now()
+	ce.users++
+}
+
+// Reduce number of references. If this returns true, there are no more users.
+// /!\ Only call this while holding the ConnPool.lock.
+func (ce *connEntry) put() {
+	ce.users--
+	// do not call ce.destroy(), let ConnPool.gc() do that
+}
+
+// Destroy a connEntry object, close the connection to the Ceph cluster.
+func (ce *connEntry) destroy() {
+	if ce.conn != nil {
+		ce.conn.Shutdown()
+		ce.conn = nil
+	}
+}

internal/util/conn_pool_test.go

@@ -0,0 +1,178 @@
+/*
+Copyright 2020 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ceph/go-ceph/rados"
+)
+
+const (
+	interval = 15 * time.Minute
+	expiry   = 10 * time.Minute
+)
+
+// fakeGet is used as a replacement for ConnPool.Get and does not need a
+// working Ceph cluster to connect to.
+//
+// This is mostly a copy of ConnPool.Get()
+func (cp *ConnPool) fakeGet(pool, monitors, user, keyfile string) (*rados.Conn, string, error) {
+	unique, err := cp.generateUniqueKey(pool, monitors, user, keyfile)
+	if err != nil {
+		return nil, "", err
+	}
+
+	// need a lock while calling ce.touch()
+	cp.lock.RLock()
+	conn := cp.getConn(unique)
+	cp.lock.RUnlock()
+	if conn != nil {
+		return conn, unique, nil
+	}
+
+	// cp.Get() creates and connects a rados.Conn here
+	conn, err = rados.NewConn()
+	if err != nil {
+		return nil, "", err
+	}
+
+	ce := &connEntry{
+		conn:     conn,
+		lastUsed: time.Now(),
+		users:    1,
+	}
+
+	cp.lock.Lock()
+	defer cp.lock.Unlock()
+	oldConn := cp.getConn(unique)
+	if oldConn != nil {
+		// there was a race, oldConn already exists
+		ce.destroy()
+		return oldConn, unique, nil
+	}
+	// this really is a new connection, add it to the map
+	cp.conns[unique] = ce
+
+	return conn, unique, nil
+}
+
+func TestConnPool(t *testing.T) {
+	cp := NewConnPool(interval, expiry)
+	defer cp.Destroy()
+
+	// create a keyfile with some contents
+	keyfile := "/tmp/conn_utils.keyfile"
+	err := ioutil.WriteFile(keyfile, []byte("the-key"), 0600)
+	if err != nil {
+		t.Errorf("failed to create keyfile: %v", err)
+		return
+	}
+	defer os.Remove(keyfile)
+
+	var conn *rados.Conn
+	var unique string
+
+	t.Run("fakeGet", func(t *testing.T) {
+		conn, unique, err = cp.fakeGet("pool", "monitors", "user", keyfile)
+		if err != nil {
+			t.Errorf("failed to get connection: %v", err)
+		}
+		// prevent goanalysis_metalinter from complaining about unused conn
+		_ = conn
+
+		// there should be a single item in cp.conns
+		if len(cp.conns) != 1 {
+			t.Errorf("there is more than a single conn in cp.conns: %v", len(cp.conns))
+		}
+
+		// the ce should have a single user
+		ce, exists := cp.conns[unique]
+		if !exists {
+			t.Errorf("getting the conn from cp.conns failed")
+		}
+		if ce.users != 1 {
+			t.Errorf("there should only be one user: %v", ce.users)
+		}
+	})
+
+	t.Run("doubleFakeGet", func(t *testing.T) {
+		// after a 2nd get, there should still be a single conn in cp.conns
+		_, _, err = cp.fakeGet("pool", "monitors", "user", keyfile)
+		if err != nil {
+			t.Errorf("failed to get connection: %v", err)
+		}
+		if len(cp.conns) != 1 {
+			t.Errorf("a second conn was added to cp.conns: %v", len(cp.conns))
+		}
+
+		// the ce should have a two users
+		ce, exists := cp.conns[unique]
+		if !exists {
+			t.Errorf("getting the conn from cp.conns failed")
+		}
+		if ce.users != 2 {
+			t.Errorf("there should be two users: %v", ce.users)
+		}
+
+		cp.Put(ce.conn)
+		if len(cp.conns) != 1 {
+			t.Errorf("a single put should not remove all cp.conns: %v", len(cp.conns))
+		}
+		// the ce should have a single user again
+		ce, exists = cp.conns[unique]
+		if !exists {
+			t.Errorf("getting the conn from cp.conns failed")
+		}
+		if ce.users != 1 {
+			t.Errorf("There should only be one user: %v", ce.users)
+		}
+	})
+
+	// there is still one conn in cp.conns after "doubleFakeGet"
+	t.Run("garbageCollection", func(t *testing.T) {
+		// timeout has not occurred yet, so number of conns in the list should stay the same
+		cp.gc()
+		if len(cp.conns) != 1 {
+			t.Errorf("gc() should not have removed any entry from cp.conns: %v", len(cp.conns))
+		}
+
+		// force expiring the ConnEntry by fetching it and adjusting .lastUsed
+		ce, exists := cp.conns[unique]
+		if !exists {
+			t.Error("getting the conn from cp.conns failed")
+		}
+		ce.lastUsed = ce.lastUsed.Add(-2 * expiry)
+
+		if ce.users != 1 {
+			t.Errorf("There should only be one user: %v", ce.users)
+		}
+		cp.Put(ce.conn)
+		if ce.users != 0 {
+			t.Errorf("There should be no users anymore: %v", ce.users)
+		}
+
+		// timeout has occurred now, so number of conns in the list should be less
+		cp.gc()
+		if len(cp.conns) != 0 {
+			t.Errorf("gc() should have removed an entry from cp.conns: %v", len(cp.conns))
+		}
+	})
+}

internal/util/credentials.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+const (
+	credUserID           = "userID"
+	credUserKey          = "userKey"
+	credAdminID          = "adminID"
+	credAdminKey         = "adminKey"
+	credMonitors         = "monitors"
+	tmpKeyFileLocation   = "/tmp/csi/keys"
+	tmpKeyFileNamePrefix = "keyfile-"
+)
+
+type Credentials struct {
+	ID      string
+	KeyFile string
+}
+
+func storeKey(key string) (string, error) {
+	tmpfile, err := ioutil.TempFile(tmpKeyFileLocation, tmpKeyFileNamePrefix)
+	if err != nil {
+		return "", fmt.Errorf("error creating a temporary keyfile (%s)", err)
+	}
+	defer func() {
+		if err != nil {
+			// don't complain about unhandled error
+			_ = os.Remove(tmpfile.Name())
+		}
+	}()
+
+	if _, err = tmpfile.Write([]byte(key)); err != nil {
+		return "", fmt.Errorf("error writing key to temporary keyfile (%s)", err)
+	}
+
+	keyFile := tmpfile.Name()
+	if keyFile == "" {
+		err = fmt.Errorf("error reading temporary filename for key (%s)", err)
+		return "", err
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		return "", fmt.Errorf("error closing temporary filename (%s)", err)
+	}
+
+	return keyFile, nil
+}
+
+func newCredentialsFromSecret(idField, keyField string, secrets map[string]string) (*Credentials, error) {
+	var (
+		c  = &Credentials{}
+		ok bool
+	)
+
+	if len(secrets) == 0 {
+		return nil, errors.New("provided secret is empty")
+	}
+	if c.ID, ok = secrets[idField]; !ok {
+		return nil, fmt.Errorf("missing ID field '%s' in secrets", idField)
+	}
+
+	key := secrets[keyField]
+	if key == "" {
+		return nil, fmt.Errorf("missing key field '%s' in secrets", keyField)
+	}
+
+	keyFile, err := storeKey(key)
+	if err == nil {
+		c.KeyFile = keyFile
+	}
+
+	return c, err
+}
+
+func (cr *Credentials) DeleteCredentials() {
+	// don't complain about unhandled error
+	_ = os.Remove(cr.KeyFile)
+}
+
+func NewUserCredentials(secrets map[string]string) (*Credentials, error) {
+	return newCredentialsFromSecret(credUserID, credUserKey, secrets)
+}
+
+func NewAdminCredentials(secrets map[string]string) (*Credentials, error) {
+	return newCredentialsFromSecret(credAdminID, credAdminKey, secrets)
+}
+
+func NewCredentials(id, key string) (*Credentials, error) {
+	var c = &Credentials{}
+
+	c.ID = id
+	keyFile, err := storeKey(key)
+	if err == nil {
+		c.KeyFile = keyFile
+	}
+
+	return c, err
+}
+
+func GetMonValFromSecret(secrets map[string]string) (string, error) {
+	if mons, ok := secrets[credMonitors]; ok {
+		return mons, nil
+	}
+	return "", fmt.Errorf("missing %q", credMonitors)
+}

internal/util/crypto.go

@@ -0,0 +1,274 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"crypto/rand"
+
+	"k8s.io/klog"
+)
+
+const (
+	mapperFilePrefix     = "luks-rbd-"
+	mapperFilePathPrefix = "/dev/mapper"
+
+	// image metadata key for encryption
+	encryptionMetaKey = ".rbd.csi.ceph.com/encrypted"
+
+	// Encryption passphrase location in K8s secrets
+	encryptionPassphraseKey = "encryptionPassphrase"
+	kmsTypeKey              = "encryptionKMSType"
+
+	// Default KMS type
+	defaultKMSType = "default"
+
+	// kmsConfigPath is the location of the vault config file
+	kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json"
+
+	// Passphrase size - 20 bytes is 160 bits to satisfy:
+	// https://tools.ietf.org/html/rfc6749#section-10.10
+	encryptionPassphraseSize = 20
+)
+
+// EncryptionKMS provides external Key Management System for encryption
+// passphrases storage
+type EncryptionKMS interface {
+	GetPassphrase(key string) (string, error)
+	SavePassphrase(key, value string) error
+	DeletePassphrase(key string) error
+	GetID() string
+}
+
+// MissingPassphrase is an error instructing to generate new passphrase
+type MissingPassphrase struct {
+	error
+}
+
+// SecretsKMS is default KMS implementation that means no KMS is in use
+type SecretsKMS struct {
+	passphrase string
+}
+
+func initSecretsKMS(secrets map[string]string) (EncryptionKMS, error) {
+	passphraseValue, ok := secrets[encryptionPassphraseKey]
+	if !ok {
+		return nil, errors.New("missing encryption passphrase in secrets")
+	}
+	return SecretsKMS{passphrase: passphraseValue}, nil
+}
+
+// GetPassphrase returns passphrase from Kubernetes secrets
+func (kms SecretsKMS) GetPassphrase(key string) (string, error) {
+	return kms.passphrase, nil
+}
+
+// SavePassphrase is not implemented
+func (kms SecretsKMS) SavePassphrase(key, value string) error {
+	return fmt.Errorf("save new passphrase is not implemented for Kubernetes secrets")
+}
+
+// DeletePassphrase is doing nothing as no new passphrases are saved with
+// SecretsKMS
+func (kms SecretsKMS) DeletePassphrase(key string) error {
+	return nil
+}
+
+// GetID is returning ID representing default KMS `default`
+func (kms SecretsKMS) GetID() string {
+	return defaultKMSType
+}
+
+// GetKMS returns an instance of Key Management System
+func GetKMS(kmsID string, secrets map[string]string) (EncryptionKMS, error) {
+	if kmsID == "" || kmsID == defaultKMSType {
+		return initSecretsKMS(secrets)
+	}
+
+	// #nosec
+	content, err := ioutil.ReadFile(kmsConfigPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read kms configuration from %s: %s",
+			kmsConfigPath, err)
+	}
+
+	var config map[string]interface{}
+	err = json.Unmarshal(content, &config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse kms configuration: %s", err)
+	}
+
+	kmsConfigData, ok := config[kmsID].(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("missing encryption KMS configuration with %s", kmsID)
+	}
+	kmsConfig := make(map[string]string)
+	for key, value := range kmsConfigData {
+		kmsConfig[key], ok = value.(string)
+		if !ok {
+			return nil, fmt.Errorf("broken KMS config: '%s' for '%s' is not a string",
+				value, key)
+		}
+	}
+
+	kmsType, ok := kmsConfig[kmsTypeKey]
+	if !ok {
+		return nil, fmt.Errorf("encryption KMS configuration for %s is missing KMS type", kmsID)
+	}
+
+	if kmsType == "vault" {
+		return InitVaultKMS(kmsID, kmsConfig, secrets)
+	}
+	return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType)
+}
+
+// GetCryptoPassphrase Retrieves passphrase to encrypt volume
+func GetCryptoPassphrase(ctx context.Context, volumeID string, kms EncryptionKMS) (string, error) {
+	passphrase, err := kms.GetPassphrase(volumeID)
+	if err == nil {
+		return passphrase, nil
+	}
+	if _, ok := err.(MissingPassphrase); ok {
+		klog.V(4).Infof(Log(ctx, "Encryption passphrase is missing for %s. Generating a new one"),
+			volumeID)
+		passphrase, err = generateNewEncryptionPassphrase()
+		if err != nil {
+			return "", fmt.Errorf("failed to generate passphrase for %s: %s", volumeID, err)
+		}
+		err = kms.SavePassphrase(volumeID, passphrase)
+		if err != nil {
+			return "", fmt.Errorf("failed to save the passphrase for %s: %s", volumeID, err)
+		}
+		return passphrase, nil
+	}
+	klog.Errorf(Log(ctx, "failed to get encryption passphrase for %s: %s"), volumeID, err)
+	return "", err
+}
+
+// generateNewEncryptionPassphrase generates a random passphrase for encryption
+func generateNewEncryptionPassphrase() (string, error) {
+	bytesPassphrase := make([]byte, encryptionPassphraseSize)
+	_, err := rand.Read(bytesPassphrase)
+	if err != nil {
+		return "", err
+	}
+	return base64.URLEncoding.EncodeToString(bytesPassphrase), nil
+}
+
+// VolumeMapper returns file name and it's path to where encrypted device should be open
+func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
+	mapperFile = mapperFilePrefix + volumeID
+	mapperFilePath = path.Join(mapperFilePathPrefix, mapperFile)
+	return mapperFile, mapperFilePath
+}
+
+// EncryptVolume encrypts provided device with LUKS
+func EncryptVolume(ctx context.Context, devicePath, passphrase string) error {
+	klog.V(4).Infof(Log(ctx, "Encrypting device %s with LUKS"), devicePath)
+	if _, _, err := LuksFormat(devicePath, passphrase); err != nil {
+		return errors.Wrapf(err, "failed to encrypt device %s with LUKS", devicePath)
+	}
+	return nil
+}
+
+// OpenEncryptedVolume opens volume so that it can be used by the client
+func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error {
+	klog.V(4).Infof(Log(ctx, "Opening device %s with LUKS on %s"), devicePath, mapperFile)
+	_, _, err := LuksOpen(devicePath, mapperFile, passphrase)
+	return err
+}
+
+// CloseEncryptedVolume closes encrypted volume so it can be detached
+func CloseEncryptedVolume(ctx context.Context, mapperFile string) error {
+	klog.V(4).Infof(Log(ctx, "Closing LUKS device %s"), mapperFile)
+	_, _, err := LuksClose(mapperFile)
+	return err
+}
+
+// IsDeviceOpen determines if encrypted device is already open
+func IsDeviceOpen(ctx context.Context, device string) (bool, error) {
+	_, mappedFile, err := DeviceEncryptionStatus(ctx, device)
+	return (mappedFile != ""), err
+}
+
+// DeviceEncryptionStatus looks to identify if the passed device is a LUKS mapping
+// and if so what the device is and the mapper name as used by LUKS.
+// If not, just returns the original device and an empty string.
+func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevice, mapper string, err error) {
+	if !strings.HasPrefix(devicePath, mapperFilePathPrefix) {
+		return devicePath, "", nil
+	}
+	mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/")
+	stdout, _, err := LuksStatus(mapPath)
+	if err != nil {
+		klog.V(4).Infof(Log(ctx, "device %s is not an active LUKS device: %v"), devicePath, err)
+		return devicePath, "", nil
+	}
+	lines := strings.Split(string(stdout), "\n")
+	if len(lines) < 1 {
+		return "", "", fmt.Errorf("device encryption status returned no stdout for %s", devicePath)
+	}
+	if !strings.HasSuffix(lines[0], " is active.") {
+		// Implies this is not a LUKS device
+		return devicePath, "", nil
+	}
+	for i := 1; i < len(lines); i++ {
+		kv := strings.SplitN(strings.TrimSpace(lines[i]), ":", 2)
+		if len(kv) < 1 {
+			return "", "", fmt.Errorf("device encryption status output for %s is badly formatted: %s",
+				devicePath, lines[i])
+		}
+		if strings.Compare(kv[0], "device") == 0 {
+			return strings.TrimSpace(kv[1]), mapPath, nil
+		}
+	}
+	// Identified as LUKS, but failed to identify a mapped device
+	return "", "", fmt.Errorf("mapped device not found in path %s", devicePath)
+}
+
+// CheckRbdImageEncrypted verifies if rbd image was encrypted when created
+func CheckRbdImageEncrypted(ctx context.Context, cr *Credentials, monitors, imageSpec string) (string, error) {
+	value, err := GetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey)
+	if err != nil {
+		klog.Errorf(Log(ctx, "checking image %s encrypted state metadata failed: %s"), imageSpec, err)
+		return "", err
+	}
+
+	encrypted := strings.TrimSpace(value)
+	klog.V(4).Infof(Log(ctx, "image %s encrypted state metadata reports %q"), imageSpec, encrypted)
+	return encrypted, nil
+}
+
+// SaveRbdImageEncryptionStatus sets image metadata for encryption status
+func SaveRbdImageEncryptionStatus(ctx context.Context, cr *Credentials, monitors, imageSpec, status string) error {
+	err := SetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey, status)
+	if err != nil {
+		err = fmt.Errorf("failed to save image metadata encryption status for %s: %v", imageSpec, err.Error())
+		klog.Errorf(Log(ctx, err.Error()))
+		return err
+	}
+	return nil
+}

internal/util/cryptsetup.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// LuksFormat sets up volume as an encrypted LUKS partition
+func LuksFormat(devicePath, passphrase string) (stdout, stderr []byte, err error) {
+	return execCryptsetupCommand(&passphrase, "-q", "luksFormat", "--hash", "sha256", devicePath, "-d", "/dev/stdin")
+}
+
+// LuksOpen opens LUKS encrypted partition and sets up a mapping
+func LuksOpen(devicePath, mapperFile, passphrase string) (stdout, stderr []byte, err error) {
+	return execCryptsetupCommand(&passphrase, "luksOpen", devicePath, mapperFile, "-d", "/dev/stdin")
+}
+
+// LuksClose removes existing mapping
+func LuksClose(mapperFile string) (stdout, stderr []byte, err error) {
+	return execCryptsetupCommand(nil, "luksClose", mapperFile)
+}
+
+// LuksStatus returns encryption status of a provided device
+func LuksStatus(mapperFile string) (stdout, stderr []byte, err error) {
+	return execCryptsetupCommand(nil, "status", mapperFile)
+}
+
+func execCryptsetupCommand(stdin *string, args ...string) (stdout, stderr []byte, err error) {
+	var (
+		program       = "cryptsetup"
+		cmd           = exec.Command(program, args...) // nolint: gosec, #nosec
+		sanitizedArgs = StripSecretInArgs(args)
+		stdoutBuf     bytes.Buffer
+		stderrBuf     bytes.Buffer
+	)
+
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = &stderrBuf
+	if stdin != nil {
+		cmd.Stdin = strings.NewReader(*stdin)
+	}
+
+	if err := cmd.Run(); err != nil {
+		return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+
+			" occurred while running %s args: %v", err, program, sanitizedArgs)
+	}
+
+	return stdoutBuf.Bytes(), nil, nil
+}

internal/util/csiconfig.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+)
+
+/*
+Mons returns a comma separated MON list from the csi config for the given clusterID
+Expected JSON structure in the passed in config file is,
+[
+	{
+		"clusterID": "<cluster-id>",
+		"monitors":
+			[
+				"<monitor-value>",
+				"<monitor-value>",
+				...
+			]
+	},
+	...
+]
+*/
+
+// clusterInfo strongly typed JSON spec for the above JSON structure
+type clusterInfo struct {
+	ClusterID string   `json:"clusterID"`
+	Monitors  []string `json:"monitors"`
+}
+
+func Mons(pathToConfig, clusterID string) (string, error) {
+	var config []clusterInfo
+
+	// #nosec
+	content, err := ioutil.ReadFile(pathToConfig)
+	if err != nil {
+		err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterID, err)
+		return "", err
+	}
+
+	err = json.Unmarshal(content, &config)
+	if err != nil {
+		return "", fmt.Errorf("unmarshal failed: %v. raw buffer response: %s",
+			err, string(content))
+	}
+
+	for _, cluster := range config {
+		if cluster.ClusterID == clusterID {
+			if len(cluster.Monitors) == 0 {
+				return "", fmt.Errorf("empty monitor list for cluster ID (%s) in config", clusterID)
+			}
+			return strings.Join(cluster.Monitors, ","), nil
+		}
+	}
+	return "", fmt.Errorf("missing configuration for cluster ID (%s)", clusterID)
+}

internal/util/csiconfig_test.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2019 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var basePath = "./test_artifacts"
+var csiClusters = "csi-clusters.json"
+var pathToConfig = basePath + "/" + csiClusters
+var clusterID1 = "test1"
+var clusterID2 = "test2"
+
+func cleanupTestData() {
+	os.RemoveAll(basePath)
+}
+
+// nolint: gocyclo
+func TestCSIConfig(t *testing.T) {
+	var err error
+	var data string
+	var content string
+
+	defer cleanupTestData()
+
+	err = os.MkdirAll(basePath, 0700)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as clusterid file is missing
+	_, err = Mons(pathToConfig, clusterID1)
+	if err == nil {
+		t.Errorf("Failed: expected error due to missing config")
+	}
+
+	data = ""
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as file is empty
+	content, err = Mons(pathToConfig, clusterID1)
+	if err == nil {
+		t.Errorf("Failed: want (%s), got (%s)", data, content)
+	}
+
+	data = "[{\"clusterIDBad\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}]"
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as clusterID data is malformed
+	content, err = Mons(pathToConfig, clusterID2)
+	if err == nil {
+		t.Errorf("Failed: want (%s), got (%s)", data, content)
+	}
+
+	data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitorsBad\":[\"mon1\",\"mon2\",\"mon3\"]}]"
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as monitors key is incorrect/missing
+	content, err = Mons(pathToConfig, clusterID2)
+	if err == nil {
+		t.Errorf("Failed: want (%s), got (%s)", data, content)
+	}
+
+	data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",2,\"mon3\"]}]"
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as monitor data is malformed
+	content, err = Mons(pathToConfig, clusterID2)
+	if err == nil {
+		t.Errorf("Failed: want (%s), got (%s)", data, content)
+	}
+
+	data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}]"
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should fail as clusterID is not present in config
+	content, err = Mons(pathToConfig, clusterID1)
+	if err == nil {
+		t.Errorf("Failed: want (%s), got (%s)", data, content)
+	}
+
+	// TEST: Should pass as clusterID is present in config
+	content, err = Mons(pathToConfig, clusterID2)
+	if err != nil || content != "mon1,mon2,mon3" {
+		t.Errorf("Failed: want (%s), got (%s) (%v)", "mon1,mon2,mon3", content, err)
+	}
+
+	data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}," +
+		"{\"clusterID\":\"" + clusterID1 + "\",\"monitors\":[\"mon4\",\"mon5\",\"mon6\"]}]"
+	err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
+	if err != nil {
+		t.Errorf("Test setup error %s", err)
+	}
+
+	// TEST: Should pass as clusterID is present in config
+	content, err = Mons(pathToConfig, clusterID1)
+	if err != nil || content != "mon4,mon5,mon6" {
+		t.Errorf("Failed: want (%s), got (%s) (%v)", "mon4,mon5,mon6", content, err)
+	}
+}

internal/util/errors.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+// ErrKeyNotFound is returned when requested key in omap is not found
+type ErrKeyNotFound struct {
+	keyName string
+	err     error
+}
+
+func (e ErrKeyNotFound) Error() string {
+	return e.err.Error()
+}
+
+// ErrObjectExists is returned when named omap is already present in rados
+type ErrObjectExists struct {
+	objectName string
+	err        error
+}
+
+func (e ErrObjectExists) Error() string {
+	return e.err.Error()
+}
+
+// ErrObjectNotFound is returned when named omap is not found in rados
+type ErrObjectNotFound struct {
+	oMapName string
+	err      error
+}
+
+func (e ErrObjectNotFound) Error() string {
+	return e.err.Error()
+}
+
+// ErrSnapNameConflict is generated when a requested CSI snap name already exists on RBD but with
+// different properties, and hence is in conflict with the passed in CSI volume name
+type ErrSnapNameConflict struct {
+	requestName string
+	err         error
+}
+
+func (e ErrSnapNameConflict) Error() string {
+	return e.err.Error()
+}
+
+// ErrPoolNotFound is returned when pool is not found
+type ErrPoolNotFound struct {
+	Pool string
+	Err  error
+}
+
+func (e ErrPoolNotFound) Error() string {
+	return e.Err.Error()
+}

internal/util/httpserver.go

@@ -0,0 +1,27 @@
+package util
+
+import (
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/klog"
+)
+
+// ValidateURL validates the url
+func ValidateURL(c *Config) error {
+	_, err := url.Parse(c.MetricsPath)
+	return err
+}
+
+// StartMetricsServer starts http server
+func StartMetricsServer(c *Config) {
+	addr := net.JoinHostPort(c.MetricsIP, strconv.Itoa(c.MetricsPort))
+	http.Handle(c.MetricsPath, promhttp.Handler())
+	err := http.ListenAndServe(addr, nil)
+	if err != nil {
+		klog.Fatalln(err)
+	}
+}

internal/util/idlocker.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"sync"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+const (
+	// VolumeOperationAlreadyExistsFmt string format to return for concurrent operation
+	VolumeOperationAlreadyExistsFmt = "an operation with the given Volume ID %s already exists"
+
+	// SnapshotOperationAlreadyExistsFmt string format to return for concurrent operation
+	SnapshotOperationAlreadyExistsFmt = "an operation with the given Snapshot ID %s already exists"
+)
+
+// VolumeLocks implements a map with atomic operations. It stores a set of all volume IDs
+// with an ongoing operation.
+type VolumeLocks struct {
+	locks sets.String
+	mux   sync.Mutex
+}
+
+// NewVolumeLocks returns new  VolumeLocks
+func NewVolumeLocks() *VolumeLocks {
+	return &VolumeLocks{
+		locks: sets.NewString(),
+	}
+}
+
+// TryAcquire tries to acquire the lock for operating on volumeID and returns true if successful.
+// If another operation is already using volumeID, returns false.
+func (vl *VolumeLocks) TryAcquire(volumeID string) bool {
+	vl.mux.Lock()
+	defer vl.mux.Unlock()
+	if vl.locks.Has(volumeID) {
+		return false
+	}
+	vl.locks.Insert(volumeID)
+	return true
+}
+
+func (vl *VolumeLocks) Release(volumeID string) {
+	vl.mux.Lock()
+	defer vl.mux.Unlock()
+	vl.locks.Delete(volumeID)
+}

internal/util/idlocker_test.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2019 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+)
+
+// very basic tests for the moment
+func TestIDLocker(t *testing.T) {
+	fakeID := "fake-id"
+	locks := NewVolumeLocks()
+	// acquire lock for fake-id
+	ok := locks.TryAcquire(fakeID)
+
+	if !ok {
+		t.Errorf("TryAcquire failed: want (%v), got (%v)",
+			true, ok)
+	}
+
+	// try to acquire lock  again for fake-id, as lock is already present
+	// it should fail
+	ok = locks.TryAcquire(fakeID)
+
+	if ok {
+		t.Errorf("TryAcquire failed: want (%v), got (%v)",
+			false, ok)
+	}
+
+	// release the lock for fake-id and try to get lock again, it should pass
+	locks.Release(fakeID)
+	ok = locks.TryAcquire(fakeID)
+
+	if !ok {
+		t.Errorf("TryAcquire failed: want (%v), got (%v)",
+			true, ok)
+	}
+}

internal/util/k8scmcache.go

@@ -0,0 +1,188 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8s "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+)
+
+// K8sCMCache to store metadata
+type K8sCMCache struct {
+	Client    *k8s.Clientset
+	Namespace string
+}
+
+const (
+	defaultNamespace = "default"
+
+	cmLabel   = "csi-metadata"
+	cmDataKey = "content"
+
+	csiMetadataLabelAttr = "com.ceph.ceph-csi/metadata"
+)
+
+// GetK8sNamespace returns pod namespace. if pod namespace is empty
+// it returns default namespace
+func GetK8sNamespace() string {
+	namespace := os.Getenv("POD_NAMESPACE")
+	if namespace == "" {
+		return defaultNamespace
+	}
+	return namespace
+}
+
+// NewK8sClient create kubernetes client
+func NewK8sClient() *k8s.Clientset {
+	var cfg *rest.Config
+	var err error
+	cPath := os.Getenv("KUBERNETES_CONFIG_PATH")
+	if cPath != "" {
+		cfg, err = clientcmd.BuildConfigFromFlags("", cPath)
+		if err != nil {
+			klog.Errorf("Failed to get cluster config with error: %v\n", err)
+			os.Exit(1)
+		}
+	} else {
+		cfg, err = rest.InClusterConfig()
+		if err != nil {
+			klog.Errorf("Failed to get cluster config with error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+	client, err := k8s.NewForConfig(cfg)
+	if err != nil {
+		klog.Errorf("Failed to create client with error: %v\n", err)
+		os.Exit(1)
+	}
+	return client
+}
+
+func (k8scm *K8sCMCache) getMetadataCM(resourceID string) (*v1.ConfigMap, error) {
+	cm, err := k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).Get(context.TODO(), resourceID, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return cm, nil
+}
+
+// ForAll list the metadata in configmaps and filters outs based on the pattern
+func (k8scm *K8sCMCache) ForAll(pattern string, destObj interface{}, f ForAllFunc) error {
+	listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", csiMetadataLabelAttr, cmLabel)}
+	cms, err := k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).List(context.TODO(), listOpts)
+	if err != nil {
+		return errors.Wrap(err, "k8s-cm-cache: failed to list metadata configmaps")
+	}
+
+	for i := range cms.Items {
+		data := cms.Items[i].Data[cmDataKey]
+		match, err := regexp.MatchString(pattern, cms.Items[i].ObjectMeta.Name)
+		if err != nil {
+			continue
+		}
+		if !match {
+			continue
+		}
+		if err = json.Unmarshal([]byte(data), destObj); err != nil {
+			return errors.Wrapf(err, "k8s-cm-cache: JSON unmarshaling failed for configmap %s", cms.Items[i].ObjectMeta.Name)
+		}
+		if err = f(cms.Items[i].ObjectMeta.Name); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Create stores the metadata in configmaps with identifier name
+func (k8scm *K8sCMCache) Create(identifier string, data interface{}) error {
+	cm, err := k8scm.getMetadataCM(identifier)
+	if cm != nil && err == nil {
+		klog.V(4).Infof("k8s-cm-cache: configmap %s already exists, skipping configmap creation", identifier)
+		return nil
+	}
+	dataJSON, err := json.Marshal(data)
+	if err != nil {
+		return errors.Wrapf(err, "k8s-cm-cache: JSON marshaling failed for configmap %s", identifier)
+	}
+	cm = &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      identifier,
+			Namespace: k8scm.Namespace,
+			Labels: map[string]string{
+				csiMetadataLabelAttr: cmLabel,
+			},
+		},
+		Data: map[string]string{},
+	}
+	cm.Data[cmDataKey] = string(dataJSON)
+
+	_, err = k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{})
+	if err != nil {
+		if apierrs.IsAlreadyExists(err) {
+			klog.V(4).Infof("k8s-cm-cache: configmap %s already exists", identifier)
+			return nil
+		}
+		return errors.Wrapf(err, "k8s-cm-cache: couldn't persist %s metadata as configmap", identifier)
+	}
+
+	klog.V(4).Infof("k8s-cm-cache: configmap %s successfully created", identifier)
+	return nil
+}
+
+// Get retrieves the metadata in configmaps with identifier name
+func (k8scm *K8sCMCache) Get(identifier string, data interface{}) error {
+	cm, err := k8scm.getMetadataCM(identifier)
+	if err != nil {
+		if apierrs.IsNotFound(err) {
+			return &CacheEntryNotFound{err}
+		}
+
+		return err
+	}
+	err = json.Unmarshal([]byte(cm.Data[cmDataKey]), data)
+	if err != nil {
+		return errors.Wrapf(err, "k8s-cm-cache: JSON unmarshaling failed for configmap %s", identifier)
+	}
+	return nil
+}
+
+// Delete deletes the metadata in configmaps with identifier name
+func (k8scm *K8sCMCache) Delete(identifier string) error {
+	err := k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).Delete(context.TODO(), identifier, metav1.DeleteOptions{})
+	if err != nil {
+		if apierrs.IsNotFound(err) {
+			klog.V(4).Infof("k8s-cm-cache: cannot delete missing metadata configmap %s, assuming it's already deleted", identifier)
+			return nil
+		}
+
+		return errors.Wrapf(err, "k8s-cm-cache: couldn't delete metadata configmap %s", identifier)
+	}
+	klog.V(4).Infof("k8s-cm-cache: successfully deleted metadata configmap %s", identifier)
+	return nil
+}

internal/util/log.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"fmt"
+)
+
+type contextKey string
+
+// CtxKey for context based logging
+var CtxKey = contextKey("ID")
+
+// ReqID for logging request ID
+var ReqID = contextKey("Req-ID")
+
+// Log helps in context based logging
+func Log(ctx context.Context, format string) string {
+	id := ctx.Value(CtxKey)
+	if id == nil {
+		return format
+	}
+	a := fmt.Sprintf("ID: %v ", id)
+	reqID := ctx.Value(ReqID)
+	if reqID == nil {
+		return a + format
+	}
+	a += fmt.Sprintf("Req-ID: %v ", reqID)
+	return a + format
+}

internal/util/nodecache.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2018 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog"
+)
+
+// NodeCache to store metadata
+type NodeCache struct {
+	BasePath string
+	CacheDir string
+}
+
+var errDec = errors.New("file not found")
+
+// EnsureCacheDirectory creates cache directory if not present
+func (nc *NodeCache) EnsureCacheDirectory(cacheDir string) error {
+	fullPath := path.Join(nc.BasePath, cacheDir)
+	if _, err := os.Stat(fullPath); os.IsNotExist(err) {
+		// #nosec
+		if err := os.Mkdir(fullPath, 0755); err != nil {
+			return errors.Wrapf(err, "node-cache: failed to create %s folder", fullPath)
+		}
+	}
+	return nil
+}
+
+// ForAll list the metadata in Nodecache and filters outs based on the pattern
+func (nc *NodeCache) ForAll(pattern string, destObj interface{}, f ForAllFunc) error {
+	err := nc.EnsureCacheDirectory(nc.CacheDir)
+	if err != nil {
+		return errors.Wrap(err, "node-cache: couldn't ensure cache directory exists")
+	}
+	files, err := ioutil.ReadDir(path.Join(nc.BasePath, nc.CacheDir))
+	if err != nil {
+		return errors.Wrapf(err, "node-cache: failed to read %s folder", nc.BasePath)
+	}
+	cachePath := path.Join(nc.BasePath, nc.CacheDir)
+	for _, file := range files {
+		err = decodeObj(cachePath, pattern, file, destObj)
+		if err == errDec {
+			continue
+		} else if err == nil {
+			if err = f(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name()))); err != nil {
+				return err
+			}
+		}
+		return err
+	}
+	return nil
+}
+
+func decodeObj(fpath, pattern string, file os.FileInfo, destObj interface{}) error {
+	match, err := regexp.MatchString(pattern, file.Name())
+	if err != nil || !match {
+		return errDec
+	}
+	if !strings.HasSuffix(file.Name(), ".json") {
+		return errDec
+	}
+	// #nosec
+	fp, err := os.Open(path.Join(fpath, file.Name()))
+	if err != nil {
+		klog.V(4).Infof("node-cache: open file: %s err %v", file.Name(), err)
+		return errDec
+	}
+	decoder := json.NewDecoder(fp)
+	if err = decoder.Decode(destObj); err != nil {
+		if err = fp.Close(); err != nil {
+			return errors.Wrapf(err, "failed to close file %s", file.Name())
+		}
+		return errors.Wrapf(err, "node-cache: couldn't decode file %s", file.Name())
+	}
+	return nil
+}
+
+// Create creates the metadata file in cache directory with identifier name
+func (nc *NodeCache) Create(identifier string, data interface{}) error {
+	file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
+	fp, err := os.Create(file)
+	if err != nil {
+		return errors.Wrapf(err, "node-cache: failed to create metadata storage file %s\n", file)
+	}
+
+	defer func() {
+		if err = fp.Close(); err != nil {
+			klog.Warningf("failed to close file:%s %v", fp.Name(), err)
+		}
+	}()
+
+	encoder := json.NewEncoder(fp)
+	if err = encoder.Encode(data); err != nil {
+		return errors.Wrapf(err, "node-cache: failed to encode metadata for file: %s\n", file)
+	}
+	klog.V(4).Infof("node-cache: successfully saved metadata into file: %s\n", file)
+	return nil
+}
+
+// Get retrieves the metadata from cache directory with identifier name
+func (nc *NodeCache) Get(identifier string, data interface{}) error {
+	file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
+	// #nosec
+	fp, err := os.Open(file)
+	if err != nil {
+		if os.IsNotExist(errors.Cause(err)) {
+			return &CacheEntryNotFound{err}
+		}
+
+		return errors.Wrapf(err, "node-cache: open error for %s", file)
+	}
+
+	defer func() {
+		if err = fp.Close(); err != nil {
+			klog.Warningf("failed to close file:%s %v", fp.Name(), err)
+		}
+	}()
+
+	decoder := json.NewDecoder(fp)
+	if err = decoder.Decode(data); err != nil {
+		return errors.Wrap(err, "rbd: decode error")
+	}
+
+	return nil
+}
+
+// Delete deletes the metadata file from cache directory with identifier name
+func (nc *NodeCache) Delete(identifier string) error {
+	file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
+	err := os.Remove(file)
+	if err != nil {
+		if os.IsNotExist(err) {
+			klog.V(4).Infof("node-cache: cannot delete missing metadata storage file %s, assuming it's already deleted", file)
+			return nil
+		}
+
+		return errors.Wrapf(err, "node-cache: error removing file %s", file)
+	}
+	klog.V(4).Infof("node-cache: successfully deleted metadata storage file at: %+v\n", file)
+	return nil
+}

internal/util/pidlimit.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	procCgroup    = "/proc/self/cgroup"
+	sysPidsMaxFmt = "/sys/fs/cgroup/pids%s/pids.max"
+)
+
+// return the cgouprs "pids.max" file of the current process
+//
+// find the line containing the pids group from the /proc/self/cgroup file
+// $ grep 'pids' /proc/self/cgroup
+// 7:pids:/kubepods.slice/kubepods-besteffort.slice/....scope
+// $ cat /sys/fs/cgroup/pids + *.scope + /pids.max
+func getCgroupPidsFile() (string, error) {
+	cgroup, err := os.Open(procCgroup)
+	if err != nil {
+		return "", err
+	}
+	defer cgroup.Close()
+
+	scanner := bufio.NewScanner(cgroup)
+	var slice string
+	for scanner.Scan() {
+		parts := strings.Split(scanner.Text(), ":")
+		if parts == nil || len(parts) < 3 {
+			continue
+		}
+		if parts[1] == "pids" {
+			slice = parts[2]
+			break
+		}
+	}
+	if slice == "" {
+		return "", fmt.Errorf("could not find a cgroup for 'pids'")
+	}
+
+	pidsMax := fmt.Sprintf(sysPidsMaxFmt, slice)
+	return pidsMax, nil
+}
+
+// GetPIDLimit returns the current PID limit, or an error. A value of -1
+// translates to "max".
+func GetPIDLimit() (int, error) {
+	pidsMax, err := getCgroupPidsFile()
+	if err != nil {
+		return 0, err
+	}
+
+	f, err := os.Open(pidsMax) // #nosec - intended reading from /sys/...
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	maxPidsStr, err := bufio.NewReader(f).ReadString('\n')
+	if err != nil && err != io.EOF {
+		return 0, err
+	}
+	maxPidsStr = strings.TrimRight(maxPidsStr, "\n")
+
+	maxPids := -1
+	if maxPidsStr != "max" {
+		maxPids, err = strconv.Atoi(maxPidsStr)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return maxPids, nil
+}
+
+// SetPIDLimit configures the given PID limit for the current process. A value
+// of -1 translates to "max".
+func SetPIDLimit(limit int) error {
+	limitStr := "max"
+	if limit != -1 {
+		limitStr = fmt.Sprintf("%d", limit)
+	}
+
+	pidsMax, err := getCgroupPidsFile()
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(pidsMax)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.WriteString(limitStr)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/util/pidlimit_test.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2019 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"os"
+	"testing"
+)
+
+// minimal test to check if GetPIDLimit() returns an int
+// changing the limit require root permissions, not tested
+func TestGetPIDLimit(t *testing.T) {
+	runTest := os.Getenv("CEPH_CSI_RUN_ALL_TESTS")
+	if runTest == "" {
+		t.Skip("not running test that requires root permissions and cgroup support")
+	}
+
+	limit, err := GetPIDLimit()
+
+	if err != nil {
+		t.Errorf("no error should be returned, got: %v", err)
+	}
+	if limit == 0 {
+		t.Error("a PID limit of 0 is invalid")
+	}
+
+	// this is expected to fail when not run as root
+	err = SetPIDLimit(4096)
+	if err != nil {
+		t.Log("failed to set PID limit, are you running as root?")
+	} else {
+		// in case it worked, reset to the previous value
+		err = SetPIDLimit(limit)
+		if err != nil {
+			t.Logf("failed to reset PID to original limit %d", limit)
+		}
+	}
+}

internal/util/stripsecrets.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"strings"
+)
+
+const (
+	keyArg              = "--key="
+	keyFileArg          = "--keyfile="
+	secretArg           = "secret="
+	optionsArgSeparator = ','
+	strippedKey         = "--key=***stripped***"
+	strippedKeyFile     = "--keyfile=***stripped***"
+	strippedSecret      = "secret=***stripped***"
+)
+
+// StripSecretInArgs strips values of either "--key"/"--keyfile" or "secret=".
+// `args` is left unchanged.
+// Expects only one occurrence of either "--key"/"--keyfile" or "secret=".
+func StripSecretInArgs(args []string) []string {
+	out := make([]string, len(args))
+	copy(out, args)
+
+	if !stripKey(out) {
+		stripSecret(out)
+	}
+
+	return out
+}
+
+func stripKey(out []string) bool {
+	for i := range out {
+		if strings.HasPrefix(out[i], keyArg) {
+			out[i] = strippedKey
+			return true
+		}
+
+		if strings.HasPrefix(out[i], keyFileArg) {
+			out[i] = strippedKeyFile
+			return true
+		}
+	}
+
+	return false
+}
+
+func stripSecret(out []string) bool {
+	for i := range out {
+		arg := out[i]
+		begin := strings.Index(arg, secretArg)
+
+		if begin == -1 {
+			continue
+		}
+
+		end := strings.IndexByte(arg[begin+len(secretArg):], optionsArgSeparator)
+
+		out[i] = arg[:begin] + strippedSecret
+		if end != -1 {
+			out[i] += arg[end+len(secretArg):]
+		}
+
+		return true
+	}
+
+	return false
+}

internal/util/topology.go

@@ -0,0 +1,256 @@
+/*
+Copyright 2020 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
+)
+
+const (
+	keySeparator   rune   = '/'
+	labelSeparator string = ","
+)
+
+func k8sGetNodeLabels(nodeName string) (map[string]string, error) {
+	client := NewK8sClient()
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get node (%s) information : %v", nodeName, err)
+	}
+
+	return node.GetLabels(), nil
+}
+
+// GetTopologyFromDomainLabels returns the CSI topology map, determined from
+// the domain labels and their values from the CO system
+// Expects domainLabels in arg to be in the format "[prefix/]<name>,[prefix/]<name>,...",
+func GetTopologyFromDomainLabels(domainLabels, nodeName, driverName string) (map[string]string, error) {
+	if domainLabels == "" {
+		return nil, nil
+	}
+
+	// size checks on domain label prefix
+	topologyPrefix := strings.ToLower("topology." + driverName)
+	if len(topologyPrefix) > 63 {
+		return nil, fmt.Errorf("computed topology label prefix (%s) for node exceeds length limits", topologyPrefix)
+	}
+	// driverName is validated, and we are adding a lowercase "topology." to it, so no validation for conformance
+
+	// Convert passed in labels to a map, and check for uniqueness
+	labelsToRead := strings.SplitN(domainLabels, labelSeparator, -1)
+	klog.Infof("passed in node labels for processing : %+v", labelsToRead)
+
+	labelsIn := make(map[string]bool)
+	labelCount := 0
+	for _, label := range labelsToRead {
+		// as we read the labels from k8s, and check for missing labels,
+		// no label conformance checks here
+		if _, ok := labelsIn[label]; ok {
+			return nil, fmt.Errorf("duplicate label (%s) found in domain labels", label)
+		}
+
+		labelsIn[label] = true
+		labelCount++
+	}
+
+	nodeLabels, err := k8sGetNodeLabels(nodeName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine values for requested labels from node labels
+	domainMap := make(map[string]string)
+	found := 0
+	for key, value := range nodeLabels {
+		if _, ok := labelsIn[key]; !ok {
+			continue
+		}
+		// label found split name component and store value
+		nameIdx := strings.IndexRune(key, keySeparator)
+		domain := key[nameIdx+1:]
+		domainMap[domain] = value
+		labelsIn[key] = false
+		found++
+	}
+
+	// Ensure all labels are found
+	if found != labelCount {
+		missingLabels := []string{}
+		for key, missing := range labelsIn {
+			if missing {
+				missingLabels = append(missingLabels, key)
+			}
+		}
+		return nil, fmt.Errorf("missing domain labels %v on node (%s)", missingLabels, nodeName)
+	}
+
+	klog.Infof("list of domains processed : %+v", domainMap)
+
+	topology := make(map[string]string)
+	for domain, value := range domainMap {
+		topology[topologyPrefix+"/"+domain] = value
+		// TODO: when implementing domain takeover/giveback, enable a domain value that can remain pinned to the node
+		// topology["topology."+driverName+"/"+domain+"-pinned"] = value
+	}
+
+	return topology, nil
+}
+
+type topologySegment struct {
+	DomainLabel string `json:"domainLabel"`
+	DomainValue string `json:"value"`
+}
+
+// TopologyConstrainedPool stores the pool name and a list of its associated topology domain values
+type TopologyConstrainedPool struct {
+	PoolName       string            `json:"poolName"`
+	DataPoolName   string            `json:"dataPool"`
+	DomainSegments []topologySegment `json:"domainSegments"`
+}
+
+// GetTopologyFromRequest extracts TopologyConstrainedPools and passed in accessibility constraints
+// from a CSI CreateVolume request
+func GetTopologyFromRequest(req *csi.CreateVolumeRequest) (*[]TopologyConstrainedPool, *csi.TopologyRequirement, error) {
+	var (
+		topologyPools []TopologyConstrainedPool
+	)
+
+	// check if parameters have pool configuration pertaining to topology
+	topologyPoolsStr := req.GetParameters()["topologyConstrainedPools"]
+	if topologyPoolsStr == "" {
+		return nil, nil, nil
+	}
+
+	// check if there are any accessibility requirements in the request
+	accessibilityRequirements := req.GetAccessibilityRequirements()
+	if accessibilityRequirements == nil {
+		return nil, nil, nil
+	}
+
+	// extract topology based pools configuration
+	err := json.Unmarshal([]byte(strings.Replace(topologyPoolsStr, "\n", " ", -1)), &topologyPools)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to parse JSON encoded topology constrained pools parameter (%s): %v", topologyPoolsStr, err)
+	}
+
+	return &topologyPools, accessibilityRequirements, nil
+}
+
+// MatchTopologyForPool returns the topology map, if the passed in pool matches any
+// passed in accessibility constraints
+func MatchTopologyForPool(topologyPools *[]TopologyConstrainedPool,
+	accessibilityRequirements *csi.TopologyRequirement, poolName string) (map[string]string, error) {
+	var topologyPool []TopologyConstrainedPool
+
+	if topologyPools == nil || accessibilityRequirements == nil {
+		return nil, nil
+	}
+
+	// find the pool in the list of topology based pools
+	for _, value := range *topologyPools {
+		if value.PoolName == poolName {
+			topologyPool = append(topologyPool, value)
+			break
+		}
+	}
+	if len(topologyPool) == 0 {
+		return nil, fmt.Errorf("none of the configured topology pools (%+v) matched passed in pool name (%s)",
+			topologyPools, poolName)
+	}
+
+	_, _, topology, err := FindPoolAndTopology(&topologyPool, accessibilityRequirements)
+
+	return topology, err
+}
+
+// FindPoolAndTopology loops through passed in "topologyPools" and also related
+// accessibility requirements, to determine which pool matches the requirement.
+// The return variables are, image poolname, data poolname, and topology map of
+// matched requirement
+func FindPoolAndTopology(topologyPools *[]TopologyConstrainedPool,
+	accessibilityRequirements *csi.TopologyRequirement) (string, string, map[string]string, error) {
+	if topologyPools == nil || accessibilityRequirements == nil {
+		return "", "", nil, nil
+	}
+
+	// select pool that fits first topology constraint preferred requirements
+	for _, topology := range accessibilityRequirements.GetPreferred() {
+		topologyPool := matchPoolToTopology(topologyPools, topology)
+		if topologyPool.PoolName != "" {
+			return topologyPool.PoolName, topologyPool.DataPoolName, topology.GetSegments(), nil
+		}
+	}
+
+	// If preferred mismatches, check requisite for a fit
+	for _, topology := range accessibilityRequirements.GetRequisite() {
+		topologyPool := matchPoolToTopology(topologyPools, topology)
+		if topologyPool.PoolName != "" {
+			return topologyPool.PoolName, topologyPool.DataPoolName, topology.GetSegments(), nil
+		}
+	}
+
+	return "", "", nil, fmt.Errorf("none of the topology constrained pools matched requested "+
+		"topology constraints : pools (%+v) requested topology (%+v)",
+		*topologyPools, *accessibilityRequirements)
+}
+
+// matchPoolToTopology loops through passed in pools, and for each pool checks if all
+// requested topology segments are present and match the request, returning the first pool
+// that hence matches (or an empty string if none match)
+func matchPoolToTopology(topologyPools *[]TopologyConstrainedPool, topology *csi.Topology) TopologyConstrainedPool {
+	domainMap := extractDomainsFromlabels(topology)
+
+	// check if any pool matches all the domain keys and values
+	for _, topologyPool := range *topologyPools {
+		mismatch := false
+		// match all pool topology labels to requested topology
+		for _, segment := range topologyPool.DomainSegments {
+			if domainValue, ok := domainMap[segment.DomainLabel]; !ok || domainValue != segment.DomainValue {
+				mismatch = true
+				break
+			}
+		}
+
+		if mismatch {
+			continue
+		}
+
+		return topologyPool
+	}
+
+	return TopologyConstrainedPool{}
+}
+
+// extractDomainsFromlabels returns the domain name map, from passed in domain segments,
+// which is of the form [prefix/]<name>
+func extractDomainsFromlabels(topology *csi.Topology) map[string]string {
+	domainMap := make(map[string]string)
+	for domainKey, value := range topology.GetSegments() {
+		domainIdx := strings.IndexRune(domainKey, keySeparator)
+		domain := domainKey[domainIdx+1:]
+		domainMap[domain] = value
+	}
+
+	return domainMap
+}

internal/util/topology_test.go

@@ -0,0 +1,379 @@
+/*
+Copyright 2020 ceph-csi authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+// nolint: gocyclo
+// TestFindPoolAndTopology also tests MatchTopologyForPool
+func TestFindPoolAndTopology(t *testing.T) {
+	var err error
+	var label1 = "region"
+	var label2 = "zone"
+	var l1Value1 = "R1"
+	var l1Value2 = "R2"
+	var l2Value1 = "Z1"
+	var l2Value2 = "Z2"
+	var pool1 = "PoolA"
+	var pool2 = "PoolB"
+	var topologyPrefix = "prefix"
+	var emptyTopoPools = []TopologyConstrainedPool{}
+	var emptyPoolNameTopoPools = []TopologyConstrainedPool{
+		{
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value1,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value1,
+				},
+			},
+		},
+	}
+	var emptyDomainsInTopoPools = []TopologyConstrainedPool{
+		{
+			PoolName: pool1,
+		},
+	}
+	var partialDomainsInTopoPools = []TopologyConstrainedPool{
+		{
+			PoolName: pool1,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value1,
+				},
+			},
+		},
+	}
+	var differentDomainsInTopoPools = []TopologyConstrainedPool{
+		{
+			PoolName: pool1,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1 + "fuzz1",
+					DomainValue: l1Value1,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value1,
+				},
+			},
+		},
+		{
+			PoolName: pool2,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value2,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value2 + "fuzz1",
+				},
+			},
+		},
+	}
+	var validSingletonTopoPools = []TopologyConstrainedPool{
+		{
+			PoolName: pool1,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value1,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value1,
+				},
+			},
+		},
+	}
+	var validMultipleTopoPools = []TopologyConstrainedPool{
+		{
+			PoolName: pool1,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value1,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value1,
+				},
+			},
+		},
+		{
+			PoolName: pool2,
+			DomainSegments: []topologySegment{
+				{
+					DomainLabel: label1,
+					DomainValue: l1Value2,
+				},
+				{
+					DomainLabel: label2,
+					DomainValue: l2Value2,
+				},
+			},
+		},
+	}
+	var emptyAccReq = csi.TopologyRequirement{}
+	var emptySegmentAccReq = csi.TopologyRequirement{
+		Requisite: []*csi.Topology{
+			{},
+			{},
+		},
+	}
+	var partialHigherSegmentAccReq = csi.TopologyRequirement{
+		Preferred: []*csi.Topology{
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value1,
+				},
+			},
+		},
+	}
+	var partialLowerSegmentAccReq = csi.TopologyRequirement{
+		Preferred: []*csi.Topology{
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label2: l2Value1,
+				},
+			},
+		},
+	}
+	var differentSegmentAccReq = csi.TopologyRequirement{
+		Requisite: []*csi.Topology{
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1 + "fuzz2": l1Value1,
+					topologyPrefix + "/" + label2:           l2Value1,
+				},
+			},
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value2,
+					topologyPrefix + "/" + label2: l2Value2 + "fuzz2",
+				},
+			},
+		},
+	}
+	var validAccReq = csi.TopologyRequirement{
+		Requisite: []*csi.Topology{
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value1,
+					topologyPrefix + "/" + label2: l2Value1,
+				},
+			},
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value2,
+					topologyPrefix + "/" + label2: l2Value2,
+				},
+			},
+		},
+		Preferred: []*csi.Topology{
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value1,
+					topologyPrefix + "/" + label2: l2Value1,
+				},
+			},
+			{
+				Segments: map[string]string{
+					topologyPrefix + "/" + label1: l1Value2,
+					topologyPrefix + "/" + label2: l2Value2,
+				},
+			},
+		},
+	}
+
+	checkOutput := func(err error, poolName string, topoSegment map[string]string) error {
+		if err != nil {
+			return fmt.Errorf("expected success, got err (%v)", err)
+		}
+		if poolName != pool1 || !(len(topoSegment) == 2) &&
+			topoSegment[topologyPrefix+"/"+label1] == l1Value1 &&
+			topoSegment[topologyPrefix+"/"+label2] == l2Value1 {
+			return fmt.Errorf("expected poolName (%s) and topoSegment (%s %s), got (%s) and (%v)", pool1,
+				topologyPrefix+"/"+label1+l1Value1, topologyPrefix+"/"+label2+l2Value1,
+				poolName, topoSegment)
+		}
+		return nil
+	}
+	// Test nil values
+	_, _, _, err = FindPoolAndTopology(nil, nil)
+	if err != nil {
+		t.Errorf("expected success due to nil in-args (%v)", err)
+	}
+
+	poolName, _, _, err := FindPoolAndTopology(&validMultipleTopoPools, nil)
+	if err != nil || poolName != "" {
+		t.Errorf("expected success due to nil accessibility requirements (err - %v) (poolName - %s)", err, poolName)
+	}
+
+	poolName, _, _, err = FindPoolAndTopology(nil, &validAccReq)
+	if err != nil || poolName != "" {
+		t.Errorf("expected success due to nil topology pools (err - %v) (poolName - %s)", err, poolName)
+	}
+
+	// Test valid accessibility requirement, with invalid topology pools values
+	_, _, _, err = FindPoolAndTopology(&emptyTopoPools, &validAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to empty topology pools")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&emptyPoolNameTopoPools, &validAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to missing pool name in topology pools")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&differentDomainsInTopoPools, &validAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to mismatching domains in topology pools")
+	}
+
+	// Test valid topology pools, with invalid accessibility requirements
+	_, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &emptyAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to empty accessibility requirements")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &emptySegmentAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to empty segments in accessibility requirements")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialHigherSegmentAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to partial segments in accessibility requirements")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &partialLowerSegmentAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to partial segments in accessibility requirements")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialLowerSegmentAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to partial segments in accessibility requirements")
+	}
+
+	_, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &differentSegmentAccReq)
+	if err == nil {
+		t.Errorf("expected failure due to mismatching segments in accessibility requirements")
+	}
+
+	// Test success cases
+	// If a pool is a superset of domains (either empty domain labels or partial), it can be selected
+	poolName, _, topoSegment, err := FindPoolAndTopology(&emptyDomainsInTopoPools, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+
+	poolName, _, topoSegment, err = FindPoolAndTopology(&partialDomainsInTopoPools, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+
+	// match in a singleton topology pools
+	poolName, _, topoSegment, err = FindPoolAndTopology(&validSingletonTopoPools, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+
+	// match first in multiple topology pools
+	poolName, _, topoSegment, err = FindPoolAndTopology(&validMultipleTopoPools, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+
+	// match non-first in multiple topology pools
+	switchPoolOrder := []TopologyConstrainedPool{}
+	switchPoolOrder = append(switchPoolOrder, validMultipleTopoPools[1], validMultipleTopoPools[0])
+	poolName, _, topoSegment, err = FindPoolAndTopology(&switchPoolOrder, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+
+	// test valid dataPool return
+	for i := range switchPoolOrder {
+		switchPoolOrder[i].DataPoolName = "ec-" + switchPoolOrder[i].PoolName
+	}
+	poolName, dataPoolName, topoSegment, err := FindPoolAndTopology(&switchPoolOrder, &validAccReq)
+	err = checkOutput(err, poolName, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+	if dataPoolName != "ec-"+poolName {
+		t.Errorf("expected data pool to be named ec-%s, got %s", poolName, dataPoolName)
+	}
+
+	// TEST: MatchTopologyForPool
+	// check for non-existent pool
+	_, err = MatchTopologyForPool(&validMultipleTopoPools, &validAccReq, pool1+"fuzz")
+	if err == nil {
+		t.Errorf("expected failure due to non-existent pool name (%s) got success", pool1+"fuzz")
+	}
+
+	// check for existing pool
+	topoSegment, err = MatchTopologyForPool(&validMultipleTopoPools, &validAccReq, pool1)
+	err = checkOutput(err, pool1, topoSegment)
+	if err != nil {
+		t.Errorf("expected success got: (%v)", err)
+	}
+}
+
+/*
+// TODO: To test GetTopologyFromDomainLabels we need it to accept a k8s client interface, to mock k8sGetNdeLabels output
+func TestGetTopologyFromDomainLabels(t *testing.T) {
+	fakeNodes := v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "worker1",
+			Labels: map[string]string{
+				"prefix/region": "R1",
+				"prefix/zone":   "Z1",
+			},
+		},
+	}
+
+	client := fake.NewSimpleClientset(&fakeNodes)
+
+	_, err := k8sGetNodeLabels(client, "nodeName")
+	if err == nil {
+		t.Error("Expected error due to invalid node name, got success")
+	}
+
+	labels, err := k8sGetNodeLabels(client, "worker1")
+	if err != nil {
+		t.Errorf("Expected success, got err (%v)", err)
+	}
+	t.Errorf("Read labels (%v)", labels)
+}*/

internal/util/util.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"math"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/cloud-provider/volume/helpers"
+	"k8s.io/klog"
+	"k8s.io/utils/mount"
+)
+
+// RoundOffVolSize rounds up given quantity upto chunks of MiB/GiB
+func RoundOffVolSize(size int64) int64 {
+	size = RoundOffBytes(size)
+	// convert size back to MiB for rbd CLI
+	return size / helpers.MiB
+}
+
+// RoundOffBytes converts roundoff the size
+// 1.1Mib will be round off to 2Mib same for GiB
+// size less than 1MiB will be round off to 1MiB
+func RoundOffBytes(bytes int64) int64 {
+	var num int64
+	floatBytes := float64(bytes)
+	// round off the value if its in decimal
+	if floatBytes < helpers.GiB {
+		num = int64(math.Ceil(floatBytes / helpers.MiB))
+		num *= helpers.MiB
+	} else {
+		num = int64(math.Ceil(floatBytes / helpers.GiB))
+		num *= helpers.GiB
+	}
+	return num
+}
+
+// variables which will be set during the build time
+var (
+	// GitCommit tell the latest git commit image is built from
+	GitCommit string
+	// DriverVersion which will be driver version
+	DriverVersion string
+)
+
+// Config holds the parameters list which can be configured
+type Config struct {
+	Vtype           string // driver type [rbd|cephfs|liveness]
+	Endpoint        string // CSI endpoint
+	DriverName      string // name of the driver
+	NodeID          string // node id
+	InstanceID      string // unique ID distinguishing this instance of Ceph CSI
+	MetadataStorage string // metadata persistence method [node|k8s_configmap]
+	PluginPath      string // location of cephcsi plugin
+	DomainLabels    string // list of domain labels to read from the node
+
+	// cephfs related flags
+	MountCacheDir string // mount info cache save dir
+
+	// metrics related flags
+	MetricsPath       string        // path of prometheus endpoint where metrics will be available
+	HistogramOption   string        // Histogram option for grpc metrics, should be comma separated value, ex:= "0.5,2,6" where start=0.5 factor=2, count=6
+	MetricsIP         string        // TCP port for liveness/ metrics requests
+	PidLimit          int           // PID limit to configure through cgroups")
+	MetricsPort       int           // TCP port for liveness/grpc metrics requests
+	PollTime          time.Duration // time interval in seconds between each poll
+	PoolTimeout       time.Duration // probe timeout in seconds
+	EnableGRPCMetrics bool          // option to enable grpc metrics
+
+	IsControllerServer bool // if set to true start provisoner server
+	IsNodeServer       bool // if set to true start node server
+	Version            bool // cephcsi version
+
+	// cephfs related flags
+	ForceKernelCephFS bool // force to use the ceph kernel client even if the kernel is < 4.17
+
+}
+
+// CreatePersistanceStorage creates storage path and initializes new cache
+func CreatePersistanceStorage(sPath, metaDataStore, pluginPath string) (CachePersister, error) {
+	var err error
+	if err = CreateMountPoint(path.Join(sPath, "controller")); err != nil {
+		klog.Errorf("failed to create persistent storage for controller: %v", err)
+		return nil, err
+	}
+
+	if err = CreateMountPoint(path.Join(sPath, "node")); err != nil {
+		klog.Errorf("failed to create persistent storage for node: %v", err)
+		return nil, err
+	}
+
+	cp, err := NewCachePersister(metaDataStore, pluginPath)
+	if err != nil {
+		klog.Errorf("failed to define cache persistence method: %v", err)
+		return nil, err
+	}
+	return cp, err
+}
+
+// ValidateDriverName validates the driver name
+func ValidateDriverName(driverName string) error {
+	if driverName == "" {
+		return errors.New("driver name is empty")
+	}
+
+	if len(driverName) > 63 {
+		return errors.New("driver name length should be less than 63 chars")
+	}
+	var err error
+	for _, msg := range validation.IsDNS1123Subdomain(strings.ToLower(driverName)) {
+		if err == nil {
+			err = errors.New(msg)
+			continue
+		}
+		err = errors.Wrap(err, msg)
+	}
+	return err
+}
+
+// GenerateVolID generates a volume ID based on passed in parameters and version, to be returned
+// to the CO system
+func GenerateVolID(ctx context.Context, monitors string, cr *Credentials, locationID int64, pool, clusterID, objUUID string, volIDVersion uint16) (string, error) {
+	var err error
+
+	if locationID == InvalidPoolID {
+		locationID, err = GetPoolID(ctx, monitors, cr, pool)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	// generate the volume ID to return to the CO system
+	vi := CSIIdentifier{
+		LocationID:      locationID,
+		EncodingVersion: volIDVersion,
+		ClusterID:       clusterID,
+		ObjectUUID:      objUUID,
+	}
+
+	volID, err := vi.ComposeCSIID()
+
+	return volID, err
+}
+
+// CreateMountPoint creates the directory with given path
+func CreateMountPoint(mountPath string) error {
+	return os.MkdirAll(mountPath, 0750)
+}
+
+// checkDirExists checks directory  exists or not
+func checkDirExists(p string) bool {
+	if _, err := os.Stat(p); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+// IsMountPoint checks if the given path is mountpoint or not
+func IsMountPoint(p string) (bool, error) {
+	dummyMount := mount.New("")
+	notMnt, err := dummyMount.IsLikelyNotMountPoint(p)
+	if err != nil {
+		return false, status.Error(codes.Internal, err.Error())
+	}
+
+	return !notMnt, nil
+}
+
+// Mount mounts the source to target path
+func Mount(source, target, fstype string, options []string) error {
+	dummyMount := mount.New("")
+	return dummyMount.Mount(source, target, fstype, options)
+}

internal/util/util_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package util
+
+import (
+	"testing"
+)
+
+func TestRoundOffBytes(t *testing.T) {
+	type args struct {
+		bytes int64
+	}
+	tests := []struct {
+		name string
+		args args
+		want int64
+	}{
+		{
+			"1MiB conversions",
+			args{
+				bytes: 1048576,
+			},
+			1048576,
+		},
+		{
+			"1000kiB conversion",
+			args{
+				bytes: 1000,
+			},
+			1048576, // equal to 1MiB
+		},
+		{
+			"1.5Mib conversion",
+			args{
+				bytes: 1572864,
+			},
+			2097152, // equal to 2MiB
+		},
+		{
+			"1.1MiB conversion",
+			args{
+				bytes: 1153434,
+			},
+			2097152, // equal to 2MiB
+		},
+		{
+			"1.5GiB conversion",
+			args{
+				bytes: 1610612736,
+			},
+			2147483648, // equal to 2GiB
+		},
+		{
+			"1.1GiB conversion",
+			args{
+				bytes: 1181116007,
+			},
+			2147483648, // equal to 2GiB
+		},
+	}
+	for _, tt := range tests {
+		ts := tt
+		t.Run(ts.name, func(t *testing.T) {
+			if got := RoundOffBytes(ts.args.bytes); got != ts.want {
+				t.Errorf("RoundOffBytes() = %v, want %v", got, ts.want)
+			}
+		})
+	}
+}
+
+func TestRoundOffVolSize(t *testing.T) {
+	type args struct {
+		size int64
+	}
+	tests := []struct {
+		name string
+		args args
+		want int64
+	}{
+		{
+			"1MiB conversions",
+			args{
+				size: 1048576,
+			},
+			1, // MiB
+		},
+		{
+			"1000kiB conversion",
+			args{
+				size: 1000,
+			},
+			1, // MiB
+		},
+		{
+			"1.5Mib conversion",
+			args{
+				size: 1572864,
+			},
+			2, // MiB
+		},
+		{
+			"1.1MiB conversion",
+			args{
+				size: 1153434,
+			},
+			2, // MiB
+		},
+		{
+			"1.5GiB conversion",
+			args{
+				size: 1610612736,
+			},
+			2048, // MiB
+		},
+		{
+			"1.1GiB conversion",
+			args{
+				size: 1181116007,
+			},
+			2048, // MiB
+		},
+	}
+	for _, tt := range tests {
+		ts := tt
+		t.Run(ts.name, func(t *testing.T) {
+			if got := RoundOffVolSize(ts.args.size); got != ts.want {
+				t.Errorf("RoundOffVolSize() = %v, want %v", got, ts.want)
+			}
+		})
+	}
+}

internal/util/validate.go

@@ -0,0 +1,80 @@
+package util
+
+import (
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ValidateNodeStageVolumeRequest validates the node stage request
+func ValidateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
+	if req.GetVolumeCapability() == nil {
+		return status.Error(codes.InvalidArgument, "volume capability missing in request")
+	}
+
+	if req.GetVolumeId() == "" {
+		return status.Error(codes.InvalidArgument, "volume ID missing in request")
+	}
+
+	if req.GetStagingTargetPath() == "" {
+		return status.Error(codes.InvalidArgument, "staging target path missing in request")
+	}
+
+	if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
+		return status.Error(codes.InvalidArgument, "stage secrets cannot be nil or empty")
+	}
+
+	// validate stagingpath exists
+	ok := checkDirExists(req.GetStagingTargetPath())
+	if !ok {
+		return status.Error(codes.InvalidArgument, "staging path does not exists on node")
+	}
+	return nil
+}
+
+// ValidateNodeUnstageVolumeRequest validates the node unstage request
+func ValidateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
+	if req.GetVolumeId() == "" {
+		return status.Error(codes.InvalidArgument, "volume ID missing in request")
+	}
+
+	if req.GetStagingTargetPath() == "" {
+		return status.Error(codes.InvalidArgument, "staging target path missing in request")
+	}
+
+	return nil
+}
+
+// ValidateNodePublishVolumeRequest validates the node publish request
+func ValidateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
+	if req.GetVolumeCapability() == nil {
+		return status.Error(codes.InvalidArgument, "volume capability missing in request")
+	}
+
+	if req.GetVolumeId() == "" {
+		return status.Error(codes.InvalidArgument, "volume ID missing in request")
+	}
+
+	if req.GetTargetPath() == "" {
+		return status.Error(codes.InvalidArgument, "target path missing in request")
+	}
+
+	if req.GetStagingTargetPath() == "" {
+		return status.Error(codes.InvalidArgument, "staging target path missing in request")
+	}
+
+	return nil
+}
+
+// ValidateNodeUnpublishVolumeRequest validates the node unpublish request
+func ValidateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
+	if req.GetVolumeId() == "" {
+		return status.Error(codes.InvalidArgument, "volume ID missing in request")
+	}
+
+	if req.GetTargetPath() == "" {
+		return status.Error(codes.InvalidArgument, "target path missing in request")
+	}
+
+	return nil
+}

internal/util/vault.go

@@ -0,0 +1,337 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+const (
+	// path to service account token that will be used to authenticate with Vault
+	// #nosec
+	serviceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+	// vault configuration defaults
+	vaultDefaultAuthPath       = "/v1/auth/kubernetes/login"
+	vaultDefaultRole           = "csi-kubernetes"
+	vaultDefaultNamespace      = ""
+	vaultDefaultPassphraseRoot = "/v1/secret"
+	vaultDefaultPassphrasePath = ""
+
+	// vault request headers
+	vaultTokenHeader     = "X-Vault-Token" // nolint: gosec, #nosec
+	vaultNamespaceHeader = "X-Vault-Namespace"
+)
+
+/*
+VaultKMS represents a Hashicorp Vault KMS configuration
+
+Example JSON structure in the KMS config is,
+{
+	"local_vault_unique_identifier": {
+		"encryptionKMSType": "vault",
+		"vaultAddress": "https://127.0.0.1:8500",
+		"vaultAuthPath": "/v1/auth/kubernetes/login",
+		"vaultRole": "csi-kubernetes",
+		"vaultNamespace": "",
+		"vaultPassphraseRoot": "/v1/secret",
+		"vaultPassphrasePath": "",
+		"vaultCAVerify": true,
+		"vaultCAFromSecret": "vault-ca"
+	},
+	...
+}
+*/
+type VaultKMS struct {
+	EncryptionKMSID     string
+	VaultAddress        string
+	VaultAuthPath       string
+	VaultRole           string
+	VaultNamespace      string
+	VaultPassphraseRoot string
+	VaultPassphrasePath string
+	VaultCAVerify       bool
+	vaultCA             *x509.CertPool
+}
+
+// InitVaultKMS returns an interface to HashiCorp Vault KMS
+func InitVaultKMS(kmsID string, config, secrets map[string]string) (EncryptionKMS, error) {
+	var (
+		ok  bool
+		err error
+	)
+	kms := &VaultKMS{}
+	kms.EncryptionKMSID = kmsID
+
+	kms.VaultAddress, ok = config["vaultAddress"]
+	if !ok || kms.VaultAddress == "" {
+		return nil, fmt.Errorf("missing 'vaultAddress' for vault KMS %s", kmsID)
+	}
+	kms.VaultAuthPath, ok = config["vaultAuthPath"]
+	if !ok || kms.VaultAuthPath == "" {
+		kms.VaultAuthPath = vaultDefaultAuthPath
+	}
+	kms.VaultRole, ok = config["vaultRole"]
+	if !ok || kms.VaultRole == "" {
+		kms.VaultRole = vaultDefaultRole
+	}
+	kms.VaultNamespace, ok = config["vaultNamespace"]
+	if !ok || kms.VaultNamespace == "" {
+		kms.VaultNamespace = vaultDefaultNamespace
+	}
+	kms.VaultPassphraseRoot, ok = config["vaultPassphraseRoot"]
+	if !ok || kms.VaultPassphraseRoot == "" {
+		kms.VaultPassphraseRoot = vaultDefaultPassphraseRoot
+	}
+	kms.VaultPassphrasePath, ok = config["vaultPassphrasePath"]
+	if !ok || kms.VaultPassphrasePath == "" {
+		kms.VaultPassphrasePath = vaultDefaultPassphrasePath
+	}
+	kms.VaultCAVerify = true
+	verifyCA, ok := config["vaultCAVerify"]
+	if ok {
+		kms.VaultCAVerify, err = strconv.ParseBool(verifyCA)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse 'vaultCAVerify' for vault <%s> kms config: %s",
+				kmsID, err)
+		}
+	}
+	vaultCAFromSecret, ok := config["vaultCAFromSecret"]
+	if ok && vaultCAFromSecret != "" {
+		caPEM, ok := secrets[vaultCAFromSecret]
+		if !ok {
+			return nil, fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
+		}
+		roots := x509.NewCertPool()
+		ok = roots.AppendCertsFromPEM([]byte(caPEM))
+		if !ok {
+			return nil, fmt.Errorf("failed loading CA bundle for vault from secret %s",
+				vaultCAFromSecret)
+		}
+		kms.vaultCA = roots
+	}
+
+	return kms, nil
+}
+
+// GetID is returning correlation ID to KMS configuration
+func (kms *VaultKMS) GetID() string {
+	return kms.EncryptionKMSID
+}
+
+// GetPassphrase returns passphrase from Vault
+func (kms *VaultKMS) GetPassphrase(key string) (string, error) {
+	var passphrase string
+	resp, err := kms.request("GET", kms.getKeyDataURI(key), nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to retrieve passphrase for %s from vault: %s",
+			key, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 404 {
+		return "", MissingPassphrase{fmt.Errorf("passphrase for %s not found", key)}
+	}
+	err = kms.processError(resp, fmt.Sprintf("get passphrase for %s", key))
+	if err != nil {
+		return "", err
+	}
+
+	// parse resp as JSON and retrieve vault token
+	var result map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return "", fmt.Errorf("failed parsing passphrase for %s from response: %s",
+			key, err)
+	}
+	data, ok := result["data"].(map[string]interface{})
+	if !ok {
+		return "", fmt.Errorf("failed parsing data for get passphrase request for %s", key)
+	}
+	data, ok = data["data"].(map[string]interface{})
+	if !ok {
+		return "", fmt.Errorf("failed parsing data.data for get passphrase request for %s", key)
+	}
+	passphrase, ok = data["passphrase"].(string)
+	if !ok {
+		return "", fmt.Errorf("failed parsing passphrase for get passphrase request for %s", key)
+	}
+
+	return passphrase, nil
+}
+
+// SavePassphrase saves new passphrase in Vault
+func (kms *VaultKMS) SavePassphrase(key, value string) error {
+	data, err := json.Marshal(map[string]map[string]string{
+		"data": {
+			"passphrase": value,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("passphrase request data is broken: %s", err)
+	}
+
+	resp, err := kms.request("POST", kms.getKeyDataURI(key), data)
+	if err != nil {
+		return fmt.Errorf("failed to POST passphrase for %s to vault: %s", key, err)
+	}
+	defer resp.Body.Close()
+	err = kms.processError(resp, "save passphrase")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// DeletePassphrase deletes passphrase from Vault
+func (kms *VaultKMS) DeletePassphrase(key string) error {
+	vaultToken, err := kms.getAccessToken()
+	if err != nil {
+		return fmt.Errorf("could not retrieve vault token to delete the passphrase at %s: %s",
+			key, err)
+	}
+
+	resp, err := kms.send("DELETE", kms.getKeyMetadataURI(key), &vaultToken, nil)
+	if err != nil {
+		return fmt.Errorf("delete passphrase at %s request to vault failed: %s", key, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 404 {
+		err = kms.processError(resp, "delete passphrase")
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (kms *VaultKMS) getKeyDataURI(key string) string {
+	return kms.VaultPassphraseRoot + "/data/" + kms.VaultPassphrasePath + key
+}
+
+func (kms *VaultKMS) getKeyMetadataURI(key string) string {
+	return kms.VaultPassphraseRoot + "/metadata/" + kms.VaultPassphrasePath + key
+}
+
+/*
+getVaultAccessToken retrieves vault token using kubernetes authentication:
+ 1. read jwt service account token from well known location
+ 2. request token from vault using service account jwt token
+Vault will verify service account jwt token with Kubernetes and return token
+if the requester is allowed
+*/
+func (kms *VaultKMS) getAccessToken() (string, error) {
+	saToken, err := ioutil.ReadFile(serviceAccountTokenPath)
+	if err != nil {
+		return "", fmt.Errorf("service account token could not be read: %s", err)
+	}
+	data, err := json.Marshal(map[string]string{
+		"role": kms.VaultRole,
+		"jwt":  string(saToken),
+	})
+	if err != nil {
+		return "", fmt.Errorf("vault token request data is broken: %s", err)
+	}
+	resp, err := kms.send("POST", kms.VaultAuthPath, nil, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to retrieve vault token: %s", err)
+	}
+	defer resp.Body.Close()
+
+	err = kms.processError(resp, "retrieve vault token")
+	if err != nil {
+		return "", err
+	}
+	// parse resp as JSON and retrieve vault token
+	var result map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return "", fmt.Errorf("failed parsing vaultToken from response: %s", err)
+	}
+
+	auth, ok := result["auth"].(map[string]interface{})
+	if !ok {
+		return "", fmt.Errorf("failed parsing vault token auth data")
+	}
+	vaultToken, ok := auth["client_token"].(string)
+	if !ok {
+		return "", fmt.Errorf("failed parsing vault client_token")
+	}
+
+	return vaultToken, nil
+}
+
+func (kms *VaultKMS) processError(resp *http.Response, action string) error {
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return nil
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to %s (%v), error body parsing failed: %s",
+			action, resp.StatusCode, err)
+	}
+	return fmt.Errorf("failed to %s (%v): %s", action, resp.StatusCode, body)
+}
+
+func (kms *VaultKMS) request(method, path string, data []byte) (*http.Response, error) {
+	vaultToken, err := kms.getAccessToken()
+	if err != nil {
+		return nil, err
+	}
+
+	return kms.send(method, path, &vaultToken, data)
+}
+
+func (kms *VaultKMS) send(method, path string, token *string, data []byte) (*http.Response, error) {
+	tlsConfig := &tls.Config{}
+	if !kms.VaultCAVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
+	if kms.vaultCA != nil {
+		tlsConfig.RootCAs = kms.vaultCA
+	}
+	netTransport := &http.Transport{TLSClientConfig: tlsConfig}
+	client := &http.Client{Transport: netTransport}
+
+	var dataToSend io.Reader
+	if data != nil {
+		dataToSend = strings.NewReader(string(data))
+	}
+
+	req, err := http.NewRequest(method, kms.VaultAddress+path, dataToSend)
+	if err != nil {
+		return nil, fmt.Errorf("could not create a Vault request: %s", err)
+	}
+
+	if kms.VaultNamespace != "" {
+		req.Header.Set(vaultNamespaceHeader, kms.VaultNamespace)
+	}
+	if token != nil {
+		req.Header.Set(vaultTokenHeader, *token)
+	}
+
+	return client.Do(req)
+}

internal/util/volid.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2019 Ceph-CSI authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"strings"
+)
+
+/*
+CSIIdentifier contains the elements that form a CSI ID to be returned by the CSI plugin, and
+contains enough information to decompose and extract required cluster and pool information to locate
+the volume that relates to the CSI ID.
+
+The CSI identifier is composed as elaborated in the comment against ComposeCSIID and thus,
+DecomposeCSIID is the inverse of the same function.
+
+The CSIIdentifier structure carries the following fields,
+- LocationID: 64 bit integer identifier determining the location of the volume on the Ceph cluster.
+  It is the ID of the poolname or fsname, for RBD or CephFS backed volumes respectively.
+- EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID,
+  and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain
+  ability to parse backward compatible encodings.
+- ClusterID: Is a unique ID per cluster that the CSI instance is serving and is restricted to
+  lengths that can be accommodated in the encoding scheme.
+- ObjectUUID: Is the on-disk uuid of the object (image/snapshot) name, for the CSI volume that
+  corresponds to this CSI ID.
+*/
+type CSIIdentifier struct {
+	LocationID      int64
+	EncodingVersion uint16
+	ClusterID       string
+	ObjectUUID      string
+}
+
+// This maximum comes from the CSI spec on max bytes allowed in the various CSI ID fields
+const maxVolIDLen = 128
+
+/*
+ComposeCSIID composes a CSI ID from passed in parameters.
+Version 1 of the encoding scheme is as follows,
+	[csi_id_version=1:4byte] + [-:1byte]
+	[length of clusterID=1:4byte] + [-:1byte]
+	[clusterID:36bytes (MAX)] + [-:1byte]
+	[poolID:16bytes] + [-:1byte]
+	[ObjectUUID:36bytes]
+
+	Total of constant field lengths, including '-' field separators would hence be,
+	4+1+4+1+1+16+1+36 = 64
+*/
+const (
+	knownFieldSize = 64
+	uuidSize       = 36
+)
+
+func (ci CSIIdentifier) ComposeCSIID() (string, error) {
+	buf16 := make([]byte, 2)
+	buf64 := make([]byte, 8)
+
+	if (knownFieldSize + len(ci.ClusterID)) > maxVolIDLen {
+		return "", errors.New("CSI ID encoding length overflow")
+	}
+
+	if len(ci.ObjectUUID) != uuidSize {
+		return "", errors.New("CSI ID invalid object uuid")
+	}
+
+	binary.BigEndian.PutUint16(buf16, ci.EncodingVersion)
+	versionEncodedHex := hex.EncodeToString(buf16)
+
+	binary.BigEndian.PutUint16(buf16, uint16(len(ci.ClusterID)))
+	clusterIDLength := hex.EncodeToString(buf16)
+
+	binary.BigEndian.PutUint64(buf64, uint64(ci.LocationID))
+	poolIDEncodedHex := hex.EncodeToString(buf64)
+
+	return strings.Join([]string{versionEncodedHex, clusterIDLength, ci.ClusterID,
+		poolIDEncodedHex, ci.ObjectUUID}, "-"), nil
+}
+
+/*
+DecomposeCSIID composes a CSIIdentifier from passed in string
+*/
+func (ci *CSIIdentifier) DecomposeCSIID(composedCSIID string) (err error) {
+	bytesToProcess := uint16(len(composedCSIID))
+
+	// if length is less that expected constant elements, then bail out!
+	if bytesToProcess < knownFieldSize {
+		return errors.New("failed to decode CSI identifier, string underflow")
+	}
+
+	buf16, err := hex.DecodeString(composedCSIID[0:4])
+	if err != nil {
+		return err
+	}
+	ci.EncodingVersion = binary.BigEndian.Uint16(buf16)
+	// 4 for version encoding and 1 for '-' separator
+	bytesToProcess -= 5
+
+	buf16, err = hex.DecodeString(composedCSIID[5:9])
+	if err != nil {
+		return err
+	}
+	clusterIDLength := binary.BigEndian.Uint16(buf16)
+	// 4 for length encoding and 1 for '-' separator
+	bytesToProcess -= 5
+
+	if bytesToProcess < (clusterIDLength + 1) {
+		return errors.New("failed to decode CSI identifier, string underflow")
+	}
+	ci.ClusterID = composedCSIID[10 : 10+clusterIDLength]
+	// additional 1 for '-' separator
+	bytesToProcess -= (clusterIDLength + 1)
+	nextFieldStartIdx := 10 + clusterIDLength + 1
+
+	if bytesToProcess < 17 {
+		return errors.New("failed to decode CSI identifier, string underflow")
+	}
+	buf64, err := hex.DecodeString(composedCSIID[nextFieldStartIdx : nextFieldStartIdx+16])
+	if err != nil {
+		return err
+	}
+	ci.LocationID = int64(binary.BigEndian.Uint64(buf64))
+	// 16 for poolID encoding and 1 for '-' separator
+	bytesToProcess -= 17
+	nextFieldStartIdx += 17
+
+	// has to be an exact match
+	if bytesToProcess != uuidSize {
+		return errors.New("failed to decode CSI identifier, string size mismatch")
+	}
+	ci.ObjectUUID = composedCSIID[nextFieldStartIdx : nextFieldStartIdx+uuidSize]
+
+	return err
+}

internal/util/volid_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2019 Ceph-CSI authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+)
+
+type testTuple struct {
+	vID           CSIIdentifier
+	composedVolID string
+	wantEnc       bool
+	wantEncError  bool
+	wantDec       bool
+	wantDecError  bool
+}
+
+// TODO: Add more test tuples to test out other edge conditions
+var testData = []testTuple{
+	{
+		vID: CSIIdentifier{
+			LocationID:      0xffff,
+			EncodingVersion: 0xffff,
+			ClusterID:       "01616094-9d93-4178-bf45-c7eac19e8b15",
+			ObjectUUID:      "00000000-1111-2222-bbbb-cacacacacaca",
+		},
+		composedVolID: "ffff-0024-01616094-9d93-4178-bf45-c7eac19e8b15-000000000000ffff-00000000-1111-2222-bbbb-cacacacacaca",
+		wantEnc:       true,
+		wantEncError:  false,
+		wantDec:       true,
+		wantDecError:  false,
+	},
+}
+
+func TestComposeDecomposeID(t *testing.T) {
+	var (
+		err           error
+		viDecompose   CSIIdentifier
+		composedVolID string
+	)
+
+	for _, test := range testData {
+		if test.wantEnc {
+			composedVolID, err = test.vID.ComposeCSIID()
+
+			if err != nil && !test.wantEncError {
+				t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
+					test, composedVolID, err)
+			}
+
+			if err == nil && test.wantEncError {
+				t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
+					test, composedVolID, err)
+			}
+
+			if !test.wantEncError && err == nil && composedVolID != test.composedVolID {
+				t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
+					test, composedVolID, err)
+			}
+		}
+
+		if test.wantDec {
+			err = viDecompose.DecomposeCSIID(test.composedVolID)
+
+			if err != nil && !test.wantDecError {
+				t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
+					test, viDecompose, err)
+			}
+
+			if err == nil && test.wantDecError {
+				t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
+					test, viDecompose, err)
+			}
+
+			if !test.wantDecError && err == nil && viDecompose != test.vID {
+				t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
+					test, viDecompose, err)
+			}
+		}
+	}
+}

internal/util/voljournal.go

@@ -0,0 +1,627 @@
+/*
+Copyright 2019 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	"github.com/pborman/uuid"
+	"github.com/pkg/errors"
+	"k8s.io/klog"
+)
+
+// Length of string representation of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 bytes
+const uuidEncodedLength = 36
+
+/*
+RADOS omaps usage:
+
+This note details how we preserve idempotent nature of create requests and retain the relationship
+between orchestrator (CO) generated names and plugin generated names for volumes and snapshots.
+
+NOTE: volume denotes an rbd image or a CephFS subvolume
+
+The implementation uses Ceph RADOS omaps to preserve the relationship between request name and
+generated volume (or snapshot) name. There are 4 types of omaps in use,
+- A "csi.volumes.[csi-id]" (or "csi.volumes"+.+CSIInstanceID), (referred to using csiDirectory variable)
+  - stores keys named using the CO generated names for volume requests (prefixed with csiNameKeyPrefix)
+  - keys are named "csi.volume."+[CO generated VolName]
+  - Key value contains the volume uuid that is created, for the CO provided name
+
+- A "csi.snaps.[csi-id]" (or "csi.snaps"+.+CSIInstanceID), (referred to using csiDirectory variable)
+  - stores keys named using the CO generated names for snapshot requests (prefixed with csiNameKeyPrefix)
+  - keys are named "csi.snap."+[CO generated SnapName]
+  - Key value contains the snapshot uuid that is created, for the CO provided name
+
+- A per volume omap named "csi.volume."+[volume uuid], (referred to as CephUUIDDirectory)
+  - stores the key named "csi.volname", that has the value of the CO generated VolName that
+  this volume refers to (referred to using csiNameKey value)
+	- stores the key named "csi.imagename", that has the value of the Ceph RBD image name
+  this volume refers to (referred to using csiImageKey value)
+
+- A per snapshot omap named "rbd.csi.snap."+[RBD snapshot uuid], (referred to as CephUUIDDirectory)
+  - stores a key named "csi.snapname", that has the value of the CO generated SnapName that this
+  snapshot refers to (referred to using csiNameKey value)
+	- stores the key named "csi.imagename", that has the value of the Ceph RBD image name
+  this snapshot refers to (referred to using csiImageKey value)
+  - stores a key named "csi.source", that has the value of the volume name that is the
+  source of the snapshot (referred to using cephSnapSourceKey value)
+
+Creation of omaps:
+When a volume create request is received (or a snapshot create, the snapshot is not detailed in this
+	comment further as the process is similar),
+- The csiDirectory is consulted to find if there is already a key with the CO VolName, and if present,
+it is used to read its references to reach the UUID that backs this VolName, to check if the
+UUID based volume can satisfy the requirements for the request
+  - If during the process of checking the same, it is found that some linking information is stale
+  or missing, the corresponding keys upto the key in the csiDirectory is cleaned up, to start afresh
+
+- If the key with the CO VolName is not found, or was cleaned up, the request is treated as a
+new create request, and an CephUUIDDirectory is created first with a generated uuid, this ensures
+that we do not use a uuid that is already in use
+
+- Next, a key with the VolName is created in the csiDirectory, and its value is updated to store the
+generated uuid
+
+- This is followed by updating the CephUUIDDirectory with the VolName in the csiNameKey and the RBD image
+name in the csiImageKey
+
+- Finally, the volume is created (or promoted from a snapshot, if content source was provided),
+using the uuid and a corresponding name prefix (namingPrefix) as the volume name
+
+The entire operation is locked based on VolName hash, to ensure there is only ever a single entity
+modifying the related omaps for a given VolName.
+
+This ensures idempotent nature of creates, as the same CO generated VolName would attempt to use
+the same volume uuid to serve the request, as the relations are saved in the respective omaps.
+
+Deletion of omaps:
+Delete requests would not contain the VolName, hence deletion uses the volume ID, which is encoded
+with the volume uuid in it, to find the volume and the CephUUIDDirectory. The CephUUIDDirectory is
+read to get the VolName that this image points to. This VolName can be further used to read and
+delete the key from the csiDirectory.
+
+As we trace back and find the VolName, we also take a hash based lock on the VolName before
+proceeding with deleting the volume and the related omap entries, to ensure there is only ever a
+single entity modifying the related omaps for a given VolName.
+*/
+
+const (
+	defaultVolumeNamingPrefix   string = "csi-vol-"
+	defaultSnapshotNamingPrefix string = "csi-snap-"
+)
+
+// CSIJournal defines the interface and the required key names for the above RADOS based OMaps
+type CSIJournal struct {
+	// csiDirectory is the name of the CSI volumes object map that contains CSI volume-name (or
+	// snapshot name) based keys
+	csiDirectory string
+
+	// CSI volume-name keyname prefix, for key in csiDirectory, suffix is the CSI passed volume name
+	csiNameKeyPrefix string
+
+	// Per Ceph volume (RBD/FS-subvolume) object map name prefix, suffix is the generated volume uuid
+	cephUUIDDirectoryPrefix string
+
+	// CSI volume-name key in per Ceph volume object map, containing CSI volume-name for which the
+	// Ceph volume was created
+	csiNameKey string
+
+	// CSI image-name key in per Ceph volume object map, containing RBD image-name
+	// of this Ceph volume
+	csiImageKey string
+
+	// pool ID where csiDirectory is maintained, as it can be different from where the ceph volume
+	// object map is maintained, during topology based provisioning
+	csiJournalPool string
+
+	// source volume name key in per Ceph snapshot object map, containing Ceph source volume uuid
+	// for which the snapshot was created
+	cephSnapSourceKey string
+
+	// namespace in which the RADOS objects are stored, default is no namespace
+	namespace string
+
+	// encryptKMS in which encryption passphrase was saved, default is no encryption
+	encryptKMSKey string
+}
+
+// NewCSIVolumeJournal returns an instance of CSIJournal for volumes
+func NewCSIVolumeJournal() *CSIJournal {
+	return &CSIJournal{
+		csiDirectory:            "csi.volumes",
+		csiNameKeyPrefix:        "csi.volume.",
+		cephUUIDDirectoryPrefix: "csi.volume.",
+		csiNameKey:              "csi.volname",
+		csiImageKey:             "csi.imagename",
+		csiJournalPool:          "csi.journalpool",
+		cephSnapSourceKey:       "",
+		namespace:               "",
+		encryptKMSKey:           "csi.volume.encryptKMS",
+	}
+}
+
+// NewCSISnapshotJournal returns an instance of CSIJournal for snapshots
+func NewCSISnapshotJournal() *CSIJournal {
+	return &CSIJournal{
+		csiDirectory:            "csi.snaps",
+		csiNameKeyPrefix:        "csi.snap.",
+		cephUUIDDirectoryPrefix: "csi.snap.",
+		csiNameKey:              "csi.snapname",
+		csiImageKey:             "csi.imagename",
+		csiJournalPool:          "csi.journalpool",
+		cephSnapSourceKey:       "csi.source",
+		namespace:               "",
+		encryptKMSKey:           "csi.volume.encryptKMS",
+	}
+}
+
+// GetNameForUUID returns volume name
+func (cj *CSIJournal) GetNameForUUID(prefix, uid string, isSnapshot bool) string {
+	if prefix == "" {
+		if isSnapshot {
+			prefix = defaultSnapshotNamingPrefix
+		} else {
+			prefix = defaultVolumeNamingPrefix
+		}
+	}
+	return prefix + uid
+}
+
+// SetCSIDirectorySuffix sets the given suffix for the csiDirectory omap
+func (cj *CSIJournal) SetCSIDirectorySuffix(suffix string) {
+	cj.csiDirectory = cj.csiDirectory + "." + suffix
+}
+
+// SetNamespace sets the namespace in which all RADOS objects would be created
+func (cj *CSIJournal) SetNamespace(ns string) {
+	cj.namespace = ns
+}
+
+// ImageData contains image name and stored CSI properties
+type ImageData struct {
+	ImageUUID       string
+	ImagePool       string
+	ImagePoolID     int64
+	ImageAttributes *ImageAttributes
+}
+
+/*
+CheckReservation checks if given request name contains a valid reservation
+- If there is a valid reservation, then the corresponding UUID for the volume/snapshot is returned
+- If there is a reservation that is stale (or not fully cleaned up), it is garbage collected using
+the UndoReservation call, as appropriate
+- If a snapshot is being checked, then its source is matched to the parentName that is provided
+
+NOTE: As the function manipulates omaps, it should be called with a lock against the request name
+held, to prevent parallel operations from modifying the state of the omaps for this request name.
+
+Return values:
+	- string: Contains the UUID that was reserved for the passed in reqName, empty if
+	there was no reservation found
+	- error: non-nil in case of any errors
+*/
+func (cj *CSIJournal) CheckReservation(ctx context.Context, monitors string, cr *Credentials,
+	journalPool, reqName, namePrefix, parentName, kmsConfig string) (*ImageData, error) {
+	var (
+		snapSource       bool
+		objUUID          string
+		savedImagePool   string
+		savedImagePoolID int64 = InvalidPoolID
+	)
+
+	if parentName != "" {
+		if cj.cephSnapSourceKey == "" {
+			err := errors.New("invalid request, cephSnapSourceKey is nil")
+			return nil, err
+		}
+		snapSource = true
+	}
+
+	// check if request name is already part of the directory omap
+	objUUIDAndPool, err := GetOMapValue(ctx, monitors, cr, journalPool, cj.namespace, cj.csiDirectory,
+		cj.csiNameKeyPrefix+reqName)
+	if err != nil {
+		// error should specifically be not found, for volume to be absent, any other error
+		// is not conclusive, and we should not proceed
+		switch err.(type) {
+		case ErrKeyNotFound, ErrPoolNotFound:
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	// check UUID only encoded value
+	if len(objUUIDAndPool) == uuidEncodedLength {
+		objUUID = objUUIDAndPool
+		savedImagePool = journalPool
+	} else { // check poolID/UUID encoding; extract the vol UUID and pool name
+		var buf64 []byte
+		components := strings.Split(objUUIDAndPool, "/")
+		objUUID = components[1]
+		savedImagePoolIDStr := components[0]
+
+		buf64, err = hex.DecodeString(savedImagePoolIDStr)
+		if err != nil {
+			return nil, err
+		}
+		savedImagePoolID = int64(binary.BigEndian.Uint64(buf64))
+
+		savedImagePool, err = GetPoolName(ctx, monitors, cr, savedImagePoolID)
+		if err != nil {
+			if _, ok := err.(ErrPoolNotFound); ok {
+				err = cj.UndoReservation(ctx, monitors, cr, journalPool, "", "", reqName)
+			}
+			return nil, err
+		}
+	}
+
+	savedImageAttributes, err := cj.GetImageAttributes(ctx, monitors, cr, savedImagePool,
+		objUUID, snapSource)
+	if err != nil {
+		// error should specifically be not found, for image to be absent, any other error
+		// is not conclusive, and we should not proceed
+		if _, ok := err.(ErrKeyNotFound); ok {
+			err = cj.UndoReservation(ctx, monitors, cr, journalPool, savedImagePool,
+				cj.GetNameForUUID(namePrefix, objUUID, snapSource), reqName)
+		}
+		return nil, err
+	}
+
+	// check if UUID key points back to the request name
+	if savedImageAttributes.RequestName != reqName {
+		// NOTE: This should never be possible, hence no cleanup, but log error
+		// and return, as cleanup may need to occur manually!
+		return nil, fmt.Errorf("internal state inconsistent, omap names mismatch,"+
+			" request name (%s) volume UUID (%s) volume omap name (%s)",
+			reqName, objUUID, savedImageAttributes.RequestName)
+	}
+
+	if kmsConfig != "" {
+		if savedImageAttributes.KmsID != kmsConfig {
+			return nil, fmt.Errorf("internal state inconsistent, omap encryption KMS"+
+				" mismatch, request KMS (%s) volume UUID (%s) volume omap KMS (%s)",
+				kmsConfig, objUUID, savedImageAttributes.KmsID)
+		}
+	}
+
+	// TODO: skipping due to excessive poolID to poolname call, also this should never happen!
+	// check if journal pool points back to the passed in journal pool
+	// if savedJournalPoolID != journalPoolID {
+
+	if snapSource {
+		// check if source UUID key points back to the parent volume passed in
+		if savedImageAttributes.SourceName != parentName {
+			// NOTE: This can happen if there is a snapname conflict, and we already have a snapshot
+			// with the same name pointing to a different UUID as the source
+			err = fmt.Errorf("snapname points to different volume, request name (%s)"+
+				" source name (%s) saved source name (%s)",
+				reqName, parentName, savedImageAttributes.SourceName)
+			return nil, ErrSnapNameConflict{reqName, err}
+		}
+	}
+
+	imageData := &ImageData{
+		ImageUUID:       objUUID,
+		ImagePool:       savedImagePool,
+		ImagePoolID:     savedImagePoolID,
+		ImageAttributes: savedImageAttributes,
+	}
+
+	return imageData, nil
+}
+
+/*
+UndoReservation undoes a reservation, in the reverse order of ReserveName
+- The UUID directory is cleaned up before the VolName key in the csiDirectory is cleaned up
+
+NOTE: Ensure that the Ceph volume (image or FS subvolume) backing the reservation is cleaned up
+prior to cleaning up the reservation
+
+NOTE: As the function manipulates omaps, it should be called with a lock against the request name
+held, to prevent parallel operations from modifying the state of the omaps for this request name.
+
+Input arguments:
+	- csiJournalPool: Pool name that holds the CSI request name based journal
+	- volJournalPool: Pool name that holds the image/subvolume and the per-image journal (may be
+	  different if image is created in a topology constrained pool)
+*/
+func (cj *CSIJournal) UndoReservation(ctx context.Context, monitors string, cr *Credentials,
+	csiJournalPool, volJournalPool, volName, reqName string) error {
+	// delete volume UUID omap (first, inverse of create order)
+
+	if volName != "" {
+		if len(volName) < 36 {
+			return fmt.Errorf("unable to parse UUID from %s, too short", volName)
+		}
+
+		imageUUID := volName[len(volName)-36:]
+		if valid := uuid.Parse(imageUUID); valid == nil {
+			return fmt.Errorf("failed parsing UUID in %s", volName)
+		}
+
+		err := RemoveObject(ctx, monitors, cr, volJournalPool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID)
+		if err != nil {
+			if _, ok := err.(ErrObjectNotFound); !ok {
+				klog.Errorf(Log(ctx, "failed removing oMap %s (%s)"), cj.cephUUIDDirectoryPrefix+imageUUID, err)
+				return err
+			}
+		}
+	}
+
+	// delete the request name key (last, inverse of create order)
+	err := RemoveOMapKey(ctx, monitors, cr, csiJournalPool, cj.namespace, cj.csiDirectory,
+		cj.csiNameKeyPrefix+reqName)
+	if err != nil {
+		klog.Errorf(Log(ctx, "failed removing oMap key %s (%s)"), cj.csiNameKeyPrefix+reqName, err)
+		return err
+	}
+
+	return err
+}
+
+// reserveOMapName creates an omap with passed in oMapNamePrefix and a generated <uuid>.
+// It ensures generated omap name does not already exist and if conflicts are detected, a set
+// number of retires with newer uuids are attempted before returning an error
+func reserveOMapName(ctx context.Context, monitors string, cr *Credentials, pool, namespace, oMapNamePrefix string) (string, error) {
+	var iterUUID string
+
+	maxAttempts := 5
+	attempt := 1
+	for attempt <= maxAttempts {
+		// generate a uuid for the image name
+		iterUUID = uuid.NewUUID().String()
+
+		err := CreateObject(ctx, monitors, cr, pool, namespace, oMapNamePrefix+iterUUID)
+		if err != nil {
+			if _, ok := err.(ErrObjectExists); ok {
+				attempt++
+				// try again with a different uuid, for maxAttempts tries
+				klog.V(4).Infof(Log(ctx, "uuid (%s) conflict detected, retrying (attempt %d of %d)"),
+					iterUUID, attempt, maxAttempts)
+				continue
+			}
+
+			return "", err
+		}
+
+		return iterUUID, nil
+	}
+
+	return "", errors.New("uuid conflicts exceeds retry threshold")
+}
+
+/*
+ReserveName adds respective entries to the csiDirectory omaps, post generating a target
+UUIDDirectory for use. Further, these functions update the UUIDDirectory omaps, to store back
+pointers to the CSI generated request names.
+
+NOTE: As the function manipulates omaps, it should be called with a lock against the request name
+held, to prevent parallel operations from modifying the state of the omaps for this request name.
+
+Input arguments:
+	- journalPool: Pool where the CSI journal is stored (maybe different than the pool where the
+	  image/subvolume is created duw to topology constraints)
+	- journalPoolID: pool ID of the journalPool
+	- imagePool: Pool where the image/subvolume is created
+	- imagePoolID: pool ID of the imagePool
+	- reqName: Name of the volume request received
+	- namePrefix: Prefix to use when generating the image/subvolume name (suffix is an auto-genetated UUID)
+	- parentName: Name of the parent image/subvolume if reservation is for a snapshot (optional)
+	- kmsConf: Name of the key management service used to encrypt the image (optional)
+
+Return values:
+	- string: Contains the UUID that was reserved for the passed in reqName
+	- string: Contains the image name that was reserved for the passed in reqName
+	- error: non-nil in case of any errors
+*/
+func (cj *CSIJournal) ReserveName(ctx context.Context, monitors string, cr *Credentials,
+	journalPool string, journalPoolID int64,
+	imagePool string, imagePoolID int64,
+	reqName, namePrefix, parentName, kmsConf string) (string, string, error) {
+	// TODO: Take in-arg as ImageAttributes?
+	var (
+		snapSource bool
+		nameKeyVal string
+	)
+
+	if parentName != "" {
+		if cj.cephSnapSourceKey == "" {
+			err := errors.New("invalid request, cephSnapSourceKey is nil")
+			return "", "", err
+		}
+		snapSource = true
+	}
+
+	// Create the UUID based omap first, to reserve the same and avoid conflicts
+	// NOTE: If any service loss occurs post creation of the UUID directory, and before
+	// setting the request name key (csiNameKey) to point back to the UUID directory, the
+	// UUID directory key will be leaked
+	volUUID, err := reserveOMapName(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix)
+	if err != nil {
+		return "", "", err
+	}
+
+	imageName := cj.GetNameForUUID(namePrefix, volUUID, snapSource)
+
+	// Create request name (csiNameKey) key in csiDirectory and store the UUID based
+	// volume name and optionally the image pool location into it
+	if journalPool != imagePool && imagePoolID != InvalidPoolID {
+		buf64 := make([]byte, 8)
+		binary.BigEndian.PutUint64(buf64, uint64(imagePoolID))
+		poolIDEncodedHex := hex.EncodeToString(buf64)
+		nameKeyVal = poolIDEncodedHex + "/" + volUUID
+	} else {
+		nameKeyVal = volUUID
+	}
+
+	err = SetOMapKeyValue(ctx, monitors, cr, journalPool, cj.namespace, cj.csiDirectory,
+		cj.csiNameKeyPrefix+reqName, nameKeyVal)
+	if err != nil {
+		return "", "", err
+	}
+	defer func() {
+		if err != nil {
+			klog.Warningf(Log(ctx, "reservation failed for volume: %s"), reqName)
+			errDefer := cj.UndoReservation(ctx, monitors, cr, imagePool, journalPool, imageName, reqName)
+			if errDefer != nil {
+				klog.Warningf(Log(ctx, "failed undoing reservation of volume: %s (%v)"), reqName, errDefer)
+			}
+		}
+	}()
+
+	// NOTE: UUID directory is stored on the same pool as the image, helps determine image attributes
+	// 	and also CSI journal pool, when only the VolumeID is passed in (e.g DeleteVolume/DeleteSnapshot,
+	// 	VolID during CreateSnapshot).
+	// Update UUID directory to store CSI request name
+	err = SetOMapKeyValue(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
+		cj.csiNameKey, reqName)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Update UUID directory to store image name
+	err = SetOMapKeyValue(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
+		cj.csiImageKey, imageName)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Update UUID directory to store encryption values
+	if kmsConf != "" {
+		err = SetOMapKeyValue(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
+			cj.encryptKMSKey, kmsConf)
+		if err != nil {
+			return "", "", err
+		}
+	}
+
+	if journalPool != imagePool && journalPoolID != InvalidPoolID {
+		buf64 := make([]byte, 8)
+		binary.BigEndian.PutUint64(buf64, uint64(journalPoolID))
+		journalPoolIDStr := hex.EncodeToString(buf64)
+
+		// Update UUID directory to store CSI journal pool name (prefer ID instead of name to be pool rename proof)
+		err = SetOMapKeyValue(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
+			cj.csiJournalPool, journalPoolIDStr)
+		if err != nil {
+			return "", "", err
+		}
+	}
+
+	if snapSource {
+		// Update UUID directory to store source volume UUID in case of snapshots
+		err = SetOMapKeyValue(ctx, monitors, cr, imagePool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
+			cj.cephSnapSourceKey, parentName)
+		if err != nil {
+			return "", "", err
+		}
+	}
+
+	return volUUID, imageName, nil
+}
+
+// ImageAttributes contains all CSI stored image attributes, typically as OMap keys
+type ImageAttributes struct {
+	RequestName   string // Contains the request name for the passed in UUID
+	SourceName    string // Contains the parent image name for the passed in UUID, if it is a snapshot
+	ImageName     string // Contains the image or subvolume name for the passed in UUID
+	KmsID         string // Contains encryption KMS, if it is an encrypted image
+	JournalPoolID int64  // Pool ID of the CSI journal pool, stored in big endian format (on-disk data)
+}
+
+// GetImageAttributes fetches all keys and their values, from a UUID directory, returning ImageAttributes structure
+func (cj *CSIJournal) GetImageAttributes(ctx context.Context, monitors string, cr *Credentials, pool, objectUUID string, snapSource bool) (*ImageAttributes, error) {
+	var (
+		err             error
+		imageAttributes *ImageAttributes = &ImageAttributes{}
+	)
+
+	if snapSource && cj.cephSnapSourceKey == "" {
+		err = errors.New("invalid request, cephSnapSourceKey is nil")
+		return nil, err
+	}
+
+	// TODO: fetch all omap vals in one call, than make multiple listomapvals
+	imageAttributes.RequestName, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
+		cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiNameKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// image key was added at some point, so not all volumes will have this key set
+	// when ceph-csi was upgraded
+	imageAttributes.ImageName, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
+		cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiImageKey)
+	if err != nil {
+		// if the key was not found, assume the default key + UUID
+		// otherwise return error
+		switch err.(type) {
+		default:
+			return nil, err
+		case ErrKeyNotFound, ErrPoolNotFound:
+		}
+
+		if snapSource {
+			imageAttributes.ImageName = defaultSnapshotNamingPrefix + objectUUID
+		} else {
+			imageAttributes.ImageName = defaultVolumeNamingPrefix + objectUUID
+		}
+	}
+
+	imageAttributes.KmsID, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
+		cj.cephUUIDDirectoryPrefix+objectUUID, cj.encryptKMSKey)
+	if err != nil {
+		// ErrKeyNotFound means no encryption KMS was used
+		switch err.(type) {
+		default:
+			return nil, fmt.Errorf("OMapVal for %s/%s failed to get encryption KMS value: %s",
+				pool, cj.cephUUIDDirectoryPrefix+objectUUID, err)
+		case ErrKeyNotFound, ErrPoolNotFound:
+		}
+	}
+
+	journalPoolIDStr, err := GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
+		cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiJournalPool)
+	if err != nil {
+		if _, ok := err.(ErrKeyNotFound); !ok {
+			return nil, err
+		}
+		imageAttributes.JournalPoolID = InvalidPoolID
+	} else {
+		var buf64 []byte
+		buf64, err = hex.DecodeString(journalPoolIDStr)
+		if err != nil {
+			return nil, err
+		}
+		imageAttributes.JournalPoolID = int64(binary.BigEndian.Uint64(buf64))
+	}
+
+	if snapSource {
+		imageAttributes.SourceName, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
+			cj.cephUUIDDirectoryPrefix+objectUUID, cj.cephSnapSourceKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return imageAttributes, nil
+}