Support snapshot feature in rbdplugin

2024-11-09 16:00:22 +00:00 · 2018-08-08 14:42:17 +09:00 · 2018-08-08 14:42:17 +09:00 · b1ccdbb154
commit b1ccdbb154
parent c93466b009
3 changed files with 483 additions and 10 deletions
--- a/pkg/rbd/controllerserver.go
+++ b/pkg/rbd/controllerserver.go
@ -18,7 +18,10 @@ package rbd

 import (
 	"fmt"
+	"os/exec"
 	"path"
+	"syscall"
+	"time"

 	"github.com/container-storage-interface/spec/lib/go/csi/v0"
 	"github.com/golang/glog"
@ -77,7 +80,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}

-	// Generating Volume Name and Volume ID, as accoeding to CSI spec they MUST be different
+	// Generating Volume Name and Volume ID, as according to CSI spec they MUST be different
 	volName := req.GetName()
 	uniqueID := uuid.NewUUID().String()
 	if len(volName) == 0 {
@ -97,14 +100,39 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	// Check if there is already RBD image with requested name
 	found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets())
 	if !found {
-		if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
-			if err != nil {
-				glog.Warningf("failed to create volume: %v", err)
+		// if VolumeContentSource is not nil, this request is for snapshot
+		if req.VolumeContentSource != nil {
+			snapshot := req.VolumeContentSource.GetSnapshot()
+			if snapshot == nil {
+				return nil, status.Error(codes.InvalidArgument, "Volume Snapshot cannot be empty")
+			}
+
+			snapshotID := snapshot.GetId()
+			if snapshotID == "" {
+				return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty")
+			}
+
+			rbdSnap := &rbdSnapshot{}
+			if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
 				return nil, err
 			}
+
+			err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets())
+			if err != nil {
+				return nil, err
+			}
+			glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName)
+		} else {
+			if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
+				if err != nil {
+					glog.Warningf("failed to create volume: %v", err)
+					return nil, err
+				}
+			}
+			glog.V(4).Infof("create volume %s", volName)
 		}
-		glog.V(4).Infof("create volume %s", volName)
 	}
+
 	// Storing volInfo into a persistent file.
 	if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
 		glog.Warningf("rbd: failed to store volInfo with error: %v", err)
@ -162,3 +190,206 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *
 func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
 	return &csi.ControllerPublishVolumeResponse{}, nil
 }
+
+func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
+		glog.Warningf("invalid create snapshot req: %v", req)
+		return nil, err
+	}
+
+	// Check sanity of request Snapshot Name, Source Volume Id
+	if len(req.Name) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Snapshot Name cannot be empty")
+	}
+	if len(req.SourceVolumeId) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty")
+	}
+
+	// Need to check for already existing snapshot name, and if found
+	// check for the requested source volume id and already allocated source volume id
+	if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil {
+		if req.SourceVolumeId == exSnap.SourceVolumeID {
+			return &csi.CreateSnapshotResponse{
+				Snapshot: &csi.Snapshot{
+					SizeBytes:      exSnap.SizeBytes,
+					Id:             exSnap.SnapID,
+					SourceVolumeId: exSnap.SourceVolumeID,
+					CreatedAt:      exSnap.CreatedAt,
+					Status: &csi.SnapshotStatus{
+						Type: csi.SnapshotStatus_READY,
+					},
+				},
+			}, nil
+		}
+		return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Snapshot with the same name: %s but with different source volume id already exist", req.GetName()))
+	}
+
+	rbdSnap, err := getRBDSnapshotOptions(req.GetParameters())
+	if err != nil {
+		return nil, err
+	}
+
+	// Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different
+	snapName := req.GetName()
+	uniqueID := uuid.NewUUID().String()
+	rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId())
+	if err != nil {
+		return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId()))
+	}
+	rbdSnap.VolName = rbdVolume.VolName
+	rbdSnap.SnapName = snapName
+	snapshotID := "csi-rbd-snapshot-" + uniqueID
+	rbdSnap.SnapID = snapshotID
+	rbdSnap.SourceVolumeID = req.GetSourceVolumeId()
+	rbdSnap.SizeBytes = rbdVolume.VolSize
+
+	err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())
+	// if we already have the snapshot, return the snapshot
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+				if status.ExitStatus() == int(syscall.EEXIST) {
+					glog.Warningf("Snapshot with the same name: %s, we return this.", req.GetName())
+				} else {
+					glog.Warningf("failed to create snapshot: %v", err)
+					return nil, err
+				}
+			} else {
+				glog.Warningf("failed to create snapshot: %v", err)
+				return nil, err
+			}
+		} else {
+			glog.Warningf("failed to create snapshot: %v", err)
+			return nil, err
+		}
+	} else {
+		glog.V(4).Infof("create snapshot %s", snapName)
+		err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())
+
+		if err != nil {
+			err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets())
+			if err != nil {
+				return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err)
+			}
+			return nil, fmt.Errorf("Snapshot is created but failed to protect snapshot")
+		}
+	}
+
+	rbdSnap.CreatedAt = time.Now().UnixNano()
+
+	// Storing snapInfo into a persistent file.
+	if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
+		glog.Warningf("rbd: failed to store sanpInfo with error: %v", err)
+		return nil, err
+	}
+	rbdSnapshots[snapshotID] = *rbdSnap
+	return &csi.CreateSnapshotResponse{
+		Snapshot: &csi.Snapshot{
+			SizeBytes:      rbdSnap.SizeBytes,
+			Id:             snapshotID,
+			SourceVolumeId: req.GetSourceVolumeId(),
+			CreatedAt:      rbdSnap.CreatedAt,
+			Status: &csi.SnapshotStatus{
+				Type: csi.SnapshotStatus_READY,
+			},
+		},
+	}, nil
+}
+
+func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
+		glog.Warningf("invalid delete snapshot req: %v", req)
+		return nil, err
+	}
+
+	snapshotID := req.GetSnapshotId()
+	if snapshotID == "" {
+		return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty")
+	}
+	rbdSnap := &rbdSnapshot{}
+	if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
+		return nil, err
+	}
+
+	// Unprotect snapshot
+	err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets())
+	if err != nil {
+		return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
+	}
+
+	// Deleting snapshot
+	glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName)
+	if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil {
+		return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
+	}
+
+	// Removing persistent storage file for the unmapped snapshot
+	if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil {
+		return nil, err
+	}
+
+	delete(rbdSnapshots, snapshotID)
+
+	return &csi.DeleteSnapshotResponse{}, nil
+}
+
+func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil {
+		glog.Warningf("invalid list snapshot req: %v", req)
+		return nil, err
+	}
+
+	sourceVolumeId := req.GetSourceVolumeId()
+
+	// TODO (sngchlko) list with token
+
+	// list only a specific snapshot which has snapshot ID
+	if snapshotID := req.GetSnapshotId(); snapshotID != "" {
+		if rbdSnap, ok := rbdSnapshots[snapshotID]; ok {
+			// if source volume ID also set, check source volume id on the cache.
+			if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId {
+				return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID))
+			}
+			return &csi.ListSnapshotsResponse{
+				Entries: []*csi.ListSnapshotsResponse_Entry{
+					{
+						Snapshot: &csi.Snapshot{
+							SizeBytes:      rbdSnap.SizeBytes,
+							Id:             rbdSnap.SnapID,
+							SourceVolumeId: rbdSnap.SourceVolumeID,
+							CreatedAt:      rbdSnap.CreatedAt,
+							Status: &csi.SnapshotStatus{
+								Type: csi.SnapshotStatus_READY,
+							},
+						},
+					},
+				},
+			}, nil
+		} else {
+			return nil, status.Error(codes.NotFound, fmt.Sprintf("Snapshot ID %s cannot found", snapshotID))
+		}
+	}
+
+	entries := []*csi.ListSnapshotsResponse_Entry{}
+	for _, rbdSnap := range rbdSnapshots {
+		// if source volume ID also set, check source volume id on the cache.
+		if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId {
+			continue
+		}
+		entries = append(entries, &csi.ListSnapshotsResponse_Entry{
+			Snapshot: &csi.Snapshot{
+				SizeBytes:      rbdSnap.SizeBytes,
+				Id:             rbdSnap.SnapID,
+				SourceVolumeId: rbdSnap.SourceVolumeID,
+				CreatedAt:      rbdSnap.CreatedAt,
+				Status: &csi.SnapshotStatus{
+					Type: csi.SnapshotStatus_READY,
+				},
+			},
+		})
+	}
+
+	return &csi.ListSnapshotsResponse{
+		Entries: entries,
+	}, nil
+}
--- a/pkg/rbd/rbd.go
+++ b/pkg/rbd/rbd.go
@ -51,25 +51,66 @@ type rbd struct {

 var (
 	rbdDriver *rbd
-	version   = "0.2.0"
+	version   = "0.3.0"
 )

 var rbdVolumes map[string]rbdVolume
+var rbdSnapshots map[string]rbdSnapshot

 // Init checks for the persistent volume file and loads all found volumes
 // into a memory structure
 func init() {
 	rbdVolumes = map[string]rbdVolume{}
+	rbdSnapshots = map[string]rbdSnapshot{}
 	if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) {
 		glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller"))
 		if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil {
 			glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err)
 		}
+	} else {
+		// Since "controller" folder exists, it means the rbdplugin has already been running, it means
+		// there might be some volumes left, they must be re-inserted into rbdVolumes map
+		loadExVolumes()
+	}
+	if _, err := os.Stat(path.Join(PluginFolder, "controller-snap")); os.IsNotExist(err) {
+		glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller-snap"))
+		if err := os.Mkdir(path.Join(PluginFolder, "controller-snap"), 0755); err != nil {
+			glog.Fatalf("Failed to create a controller's snapshots folder with error: %v\n", err)
+		}
+	} else {
+		// Since "controller-snap" folder exists, it means the rbdplugin has already been running, it means
+		// there might be some snapshots left, they must be re-inserted into rbdSnapshots map
+		loadExSnapshots()
+	}
+}
+
+// loadExSnapshots check for any *.json files in the  PluginFolder/controller-snap folder
+// and loads then into rbdSnapshots map
+func loadExSnapshots() {
+	rbdSnap := rbdSnapshot{}
+	files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller-snap"))
+	if err != nil {
+		glog.Infof("rbd: failed to read controller's snapshots folder: %s error:%v", path.Join(PluginFolder, "controller-snap"), err)
 		return
 	}
-	// Since "controller" folder exists, it means the rbdplugin has already been running, it means
-	// there might be some volumes left, they must be re-inserted into rbdVolumes map
-	loadExVolumes()
+	for _, f := range files {
+		if !strings.HasSuffix(f.Name(), ".json") {
+			continue
+		}
+		fp, err := os.Open(path.Join(PluginFolder, "controller-snap", f.Name()))
+		if err != nil {
+			glog.Infof("rbd: open file: %s err %%v", f.Name(), err)
+			continue
+		}
+		decoder := json.NewDecoder(fp)
+		if err = decoder.Decode(&rbdSnap); err != nil {
+			glog.Infof("rbd: decode file: %s err: %v", f.Name(), err)
+			fp.Close()
+			continue
+		}
+		rbdSnapshots[rbdSnap.SnapID] = rbdSnap
+	}
+	glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap"))
 }

 // loadExVolumes check for any *.json files in the  PluginFolder/controller folder
@ -78,7 +119,7 @@ func loadExVolumes() {
 	rbdVol := rbdVolume{}
 	files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller"))
 	if err != nil {
-		glog.Infof("rbd: failed to read  controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
+		glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
 		return
 	}
 	for _, f := range files {
@ -134,6 +175,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
 	rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
 		csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
+		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+		csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
 	})
 	rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})

--- a/pkg/rbd/rbd_util.go
+++ b/pkg/rbd/rbd_util.go
@ -56,6 +56,17 @@ type rbdVolume struct {
 	VolSize       int64  `json:"volSize"`
 }

+type rbdSnapshot struct {
+	SourceVolumeID string `json:"sourceVolumeID"`
+	VolName        string `json:"volName"`
+	SnapName       string `json:"snapName"`
+	SnapID         string `json:"sanpID"`
+	Monitors       string `json:"monitors"`
+	Pool           string `json:"pool"`
+	CreatedAt      int64  `json:"createdAt"`
+	SizeBytes      int64  `json:"sizeBytes"`
+}
+
 var (
 	attachdetachMutex = keymutex.NewKeyMutex()
 	supportedFeatures = sets.NewString("layering")
@ -208,6 +219,21 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
 	return rbdVol, nil
 }

+func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) {
+	var ok bool
+	rbdSnap := &rbdSnapshot{}
+	rbdSnap.Pool, ok = snapOptions["pool"]
+	if !ok {
+		return nil, fmt.Errorf("Missing required parameter pool")
+	}
+	rbdSnap.Monitors, ok = snapOptions["monitors"]
+	if !ok {
+		return nil, fmt.Errorf("Missing required parameter monitors")
+	}
+
+	return rbdSnap, nil
+}
+
 func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) {
 	var err error
 	var output []byte
@ -376,6 +402,50 @@ func deleteVolInfo(image string, persistentStoragePath string) error {
 	return nil
 }

+func persistSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error {
+	file := path.Join(persistentStoragePath, snapshot+".json")
+	fp, err := os.Create(file)
+	if err != nil {
+		glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err)
+		return fmt.Errorf("rbd: create err %s/%s", file, err)
+	}
+	defer fp.Close()
+	encoder := json.NewEncoder(fp)
+	if err = encoder.Encode(snapInfo); err != nil {
+		glog.Errorf("rbd: failed to encode snapInfo: %+v for file: %s with error: %v\n", snapInfo, file, err)
+		return fmt.Errorf("rbd: encode err: %v", err)
+	}
+	glog.Infof("rbd: successfully saved snapInfo: %+v into file: %s\n", snapInfo, file)
+	return nil
+}
+
+func loadSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error {
+	file := path.Join(persistentStoragePath, snapshot+".json")
+	fp, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("rbd: open err %s/%s", file, err)
+	}
+	defer fp.Close()
+
+	decoder := json.NewDecoder(fp)
+	if err = decoder.Decode(snapInfo); err != nil {
+		return fmt.Errorf("rbd: decode err: %v.", err)
+	}
+	return nil
+}
+
+func deleteSnapInfo(snapshot string, persistentStoragePath string) error {
+	file := path.Join(persistentStoragePath, snapshot+".json")
+	glog.Infof("rbd: Deleting file for Snapshot: %s at: %s resulting path: %+v\n", snapshot, persistentStoragePath, file)
+	err := os.Remove(file)
+	if err != nil {
+		if err != os.ErrNotExist {
+			return fmt.Errorf("rbd: error removing file: %s/%s", file, err)
+		}
+	}
+	return nil
+}
+
 func getRBDVolumeByID(volumeID string) (rbdVolume, error) {
 	if rbdVol, ok := rbdVolumes[volumeID]; ok {
 		return rbdVol, nil
@ -391,3 +461,132 @@ func getRBDVolumeByName(volName string) (rbdVolume, error) {
 	}
 	return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName)
 }
+
+func getRBDSnapshotByName(snapName string) (rbdSnapshot, error) {
+	for _, rbdSnap := range rbdSnapshots {
+		if rbdSnap.SnapName == snapName {
+			return rbdSnap, nil
+		}
+	}
+	return rbdSnapshot{}, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName)
+}
+
+func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error {
+	var output []byte
+	var err error
+
+	mon := pOpts.Monitors
+	image := pOpts.VolName
+	snapID := pOpts.SnapID
+
+	key, err := getRBDKey(RBDUserID, credentials)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
+	args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key}
+
+	output, err = execCommand("rbd", args)
+
+	if err != nil {
+		return fmt.Errorf("failed to protect snapshot: %v, command output: %s", err, string(output))
+	}
+
+	return nil
+}
+
+func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error {
+	var output []byte
+	var err error
+
+	mon := pOpts.Monitors
+	image := pOpts.VolName
+	snapID := pOpts.SnapID
+
+	key, err := getRBDKey(RBDUserID, credentials)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
+	args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key}
+
+	output, err = execCommand("rbd", args)
+
+	if err != nil {
+		return fmt.Errorf("failed to create snapshot: %v, command output: %s", err, string(output))
+	}
+
+	return nil
+}
+
+func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error {
+	var output []byte
+	var err error
+
+	mon := pOpts.Monitors
+	image := pOpts.VolName
+	snapID := pOpts.SnapID
+
+	key, err := getRBDKey(RBDUserID, credentials)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
+	args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key}
+
+	output, err = execCommand("rbd", args)
+
+	if err != nil {
+		return fmt.Errorf("failed to unprotect snapshot: %v, command output: %s", err, string(output))
+	}
+
+	return nil
+}
+
+func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error {
+	var output []byte
+	var err error
+
+	mon := pOpts.Monitors
+	image := pOpts.VolName
+	snapID := pOpts.SnapID
+
+	key, err := getRBDKey(RBDUserID, credentials)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
+	args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key}
+
+	output, err = execCommand("rbd", args)
+
+	if err != nil {
+		return fmt.Errorf("failed to delete snapshot: %v, command output: %s", err, string(output))
+	}
+
+	return nil
+}
+
+func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials map[string]string) error {
+	var output []byte
+	var err error
+
+	mon := pVolOpts.Monitors
+	image := pVolOpts.VolName
+	snapID := pSnapOpts.SnapID
+
+	key, err := getRBDKey(RBDUserID, credentials)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, RBDUserID, key)
+	args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", RBDUserID, "-m", mon, "--key=" + key}
+
+	output, err = execCommand("rbd", args)
+
+	if err != nil {
+		return fmt.Errorf("failed to restore snapshot: %v, command output: %s", err, string(output))
+	}
+
+	return nil
+}