mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-18 02:50:30 +00:00
Merge pull request #61 from sngchlko/support-snapshot-in-rbdplugin
Support snapshot in rbdplugin
This commit is contained in:
Huamin Chen
GitHub
commit
43b9f9aeaa
@ -6,3 +6,5 @@ metadata:
|
||||
data:
|
||||
# Key value corresponds to a user name defined in ceph cluster
|
||||
admin: BASE64-ENCODED-PASSWORD
|
||||
# Key value corresponds to a user name defined in ceph cluster
|
||||
kubernetes: BASE64-ENCODED-PASSWORD
|
||||
|
||||
@ -21,4 +21,8 @@ parameters:
|
||||
csiProvisionerSecretNamespace: default
|
||||
csiNodePublishSecretName: csi-rbd-secret
|
||||
csiNodePublishSecretNamespace: default
|
||||
|
||||
# Ceph users for operating RBD
|
||||
adminid: admin
|
||||
userid: kubernetes
|
||||
reclaimPolicy: Delete
|
||||
|
||||
@ -18,7 +18,10 @@ package rbd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
"github.com/golang/glog"
|
||||
@ -77,7 +80,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generating Volume Name and Volume ID, as accoeding to CSI spec they MUST be different
|
||||
// Generating Volume Name and Volume ID, as according to CSI spec they MUST be different
|
||||
volName := req.GetName()
|
||||
uniqueID := uuid.NewUUID().String()
|
||||
if len(volName) == 0 {
|
||||
@ -95,21 +98,46 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
|
||||
volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024)
|
||||
|
||||
// Check if there is already RBD image with requested name
|
||||
found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets())
|
||||
found, _, _ := rbdStatus(rbdVol, rbdVol.UserId, req.GetControllerCreateSecrets())
|
||||
if !found {
|
||||
if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
|
||||
if err != nil {
|
||||
glog.Warningf("failed to create volume: %v", err)
|
||||
// if VolumeContentSource is not nil, this request is for snapshot
|
||||
if req.VolumeContentSource != nil {
|
||||
snapshot := req.VolumeContentSource.GetSnapshot()
|
||||
if snapshot == nil {
|
||||
return nil, status.Error(codes.InvalidArgument, "Volume Snapshot cannot be empty")
|
||||
}
|
||||
|
||||
snapshotID := snapshot.GetId()
|
||||
if len(snapshotID) == 0 {
|
||||
return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty")
|
||||
}
|
||||
|
||||
rbdSnap := &rbdSnapshot{}
|
||||
if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminId, req.GetControllerCreateSecrets())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName)
|
||||
} else {
|
||||
if err := createRBDImage(rbdVol, volSizeGB, rbdVol.AdminId, req.GetControllerCreateSecrets()); err != nil {
|
||||
if err != nil {
|
||||
glog.Warningf("failed to create volume: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
glog.V(4).Infof("create volume %s", volName)
|
||||
}
|
||||
glog.V(4).Infof("create volume %s", volName)
|
||||
}
|
||||
|
||||
// Storing volInfo into a persistent file.
|
||||
if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
|
||||
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
|
||||
}
|
||||
rbdVolumes[volumeID] = *rbdVol
|
||||
rbdVolumes[volumeID] = rbdVol
|
||||
return &csi.CreateVolumeResponse{
|
||||
Volume: &csi.Volume{
|
||||
Id: volumeID,
|
||||
@ -133,7 +161,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
|
||||
volName := rbdVol.VolName
|
||||
// Deleting rbd image
|
||||
glog.V(4).Infof("deleting volume %s", volName)
|
||||
if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil {
|
||||
if err := deleteRBDImage(rbdVol, rbdVol.AdminId, req.GetControllerDeleteSecrets()); err != nil {
|
||||
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err)
|
||||
return nil, err
|
||||
}
|
||||
@ -162,3 +190,223 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *
|
||||
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
||||
return &csi.ControllerPublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
|
||||
glog.Warningf("invalid create snapshot req: %v", req)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check sanity of request Snapshot Name, Source Volume Id
|
||||
if len(req.Name) == 0 {
|
||||
return nil, status.Error(codes.InvalidArgument, "Snapshot Name cannot be empty")
|
||||
}
|
||||
if len(req.SourceVolumeId) == 0 {
|
||||
return nil, status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty")
|
||||
}
|
||||
|
||||
// Need to check for already existing snapshot name, and if found
|
||||
// check for the requested source volume id and already allocated source volume id
|
||||
if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil {
|
||||
if req.SourceVolumeId == exSnap.SourceVolumeID {
|
||||
return &csi.CreateSnapshotResponse{
|
||||
Snapshot: &csi.Snapshot{
|
||||
SizeBytes: exSnap.SizeBytes,
|
||||
Id: exSnap.SnapID,
|
||||
SourceVolumeId: exSnap.SourceVolumeID,
|
||||
CreatedAt: exSnap.CreatedAt,
|
||||
Status: &csi.SnapshotStatus{
|
||||
Type: csi.SnapshotStatus_READY,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Snapshot with the same name: %s but with different source volume id already exist", req.GetName()))
|
||||
}
|
||||
|
||||
rbdSnap, err := getRBDSnapshotOptions(req.GetParameters())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different
|
||||
snapName := req.GetName()
|
||||
uniqueID := uuid.NewUUID().String()
|
||||
rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId()))
|
||||
}
|
||||
if !hasSnapshotFeature(rbdVolume.ImageFeatures) {
|
||||
return nil, fmt.Errorf("Volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId())
|
||||
}
|
||||
|
||||
rbdSnap.VolName = rbdVolume.VolName
|
||||
rbdSnap.SnapName = snapName
|
||||
snapshotID := "csi-rbd-" + rbdVolume.VolName + "-snap-" + uniqueID
|
||||
rbdSnap.SnapID = snapshotID
|
||||
rbdSnap.SourceVolumeID = req.GetSourceVolumeId()
|
||||
rbdSnap.SizeBytes = rbdVolume.VolSize
|
||||
|
||||
err = createSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets())
|
||||
// if we already have the snapshot, return the snapshot
|
||||
if err != nil {
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() == int(syscall.EEXIST) {
|
||||
glog.Warningf("Snapshot with the same name: %s, we return this.", req.GetName())
|
||||
} else {
|
||||
glog.Warningf("failed to create snapshot: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("failed to create snapshot: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("failed to create snapshot: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
glog.V(4).Infof("create snapshot %s", snapName)
|
||||
err = protectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets())
|
||||
|
||||
if err != nil {
|
||||
err = deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err)
|
||||
}
|
||||
return nil, fmt.Errorf("Snapshot is created but failed to protect snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
rbdSnap.CreatedAt = time.Now().UnixNano()
|
||||
|
||||
// Storing snapInfo into a persistent file.
|
||||
if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
|
||||
glog.Warningf("rbd: failed to store snapInfo with error: %v", err)
|
||||
|
||||
// Unprotect snapshot
|
||||
err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
|
||||
}
|
||||
|
||||
// Deleting snapshot
|
||||
glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName)
|
||||
if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()); err != nil {
|
||||
return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
rbdSnapshots[snapshotID] = rbdSnap
|
||||
return &csi.CreateSnapshotResponse{
|
||||
Snapshot: &csi.Snapshot{
|
||||
SizeBytes: rbdSnap.SizeBytes,
|
||||
Id: snapshotID,
|
||||
SourceVolumeId: req.GetSourceVolumeId(),
|
||||
CreatedAt: rbdSnap.CreatedAt,
|
||||
Status: &csi.SnapshotStatus{
|
||||
Type: csi.SnapshotStatus_READY,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil {
|
||||
glog.Warningf("invalid delete snapshot req: %v", req)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
snapshotID := req.GetSnapshotId()
|
||||
if len(snapshotID) == 0 {
|
||||
return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty")
|
||||
}
|
||||
rbdSnap := &rbdSnapshot{}
|
||||
if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unprotect snapshot
|
||||
err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
|
||||
}
|
||||
|
||||
// Deleting snapshot
|
||||
glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName)
|
||||
if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets()); err != nil {
|
||||
return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
|
||||
}
|
||||
|
||||
// Removing persistent storage file for the unmapped snapshot
|
||||
if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
delete(rbdSnapshots, snapshotID)
|
||||
|
||||
return &csi.DeleteSnapshotResponse{}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil {
|
||||
glog.Warningf("invalid list snapshot req: %v", req)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sourceVolumeId := req.GetSourceVolumeId()
|
||||
|
||||
// TODO (sngchlko) list with token
|
||||
|
||||
// list only a specific snapshot which has snapshot ID
|
||||
if snapshotID := req.GetSnapshotId(); len(snapshotID) != 0 {
|
||||
if rbdSnap, ok := rbdSnapshots[snapshotID]; ok {
|
||||
// if source volume ID also set, check source volume id on the cache.
|
||||
if len(sourceVolumeId) != 0 && rbdSnap.SourceVolumeID != sourceVolumeId {
|
||||
return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID))
|
||||
}
|
||||
return &csi.ListSnapshotsResponse{
|
||||
Entries: []*csi.ListSnapshotsResponse_Entry{
|
||||
{
|
||||
Snapshot: &csi.Snapshot{
|
||||
SizeBytes: rbdSnap.SizeBytes,
|
||||
Id: rbdSnap.SnapID,
|
||||
SourceVolumeId: rbdSnap.SourceVolumeID,
|
||||
CreatedAt: rbdSnap.CreatedAt,
|
||||
Status: &csi.SnapshotStatus{
|
||||
Type: csi.SnapshotStatus_READY,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
} else {
|
||||
return nil, status.Error(codes.NotFound, fmt.Sprintf("Snapshot ID %s cannot found", snapshotID))
|
||||
}
|
||||
}
|
||||
|
||||
entries := []*csi.ListSnapshotsResponse_Entry{}
|
||||
for _, rbdSnap := range rbdSnapshots {
|
||||
// if source volume ID also set, check source volume id on the cache.
|
||||
if len(sourceVolumeId) != 0 && rbdSnap.SourceVolumeID != sourceVolumeId {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, &csi.ListSnapshotsResponse_Entry{
|
||||
Snapshot: &csi.Snapshot{
|
||||
SizeBytes: rbdSnap.SizeBytes,
|
||||
Id: rbdSnap.SnapID,
|
||||
SourceVolumeId: rbdSnap.SourceVolumeID,
|
||||
CreatedAt: rbdSnap.CreatedAt,
|
||||
Status: &csi.SnapshotStatus{
|
||||
Type: csi.SnapshotStatus_READY,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return &csi.ListSnapshotsResponse{
|
||||
Entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
||||
}
|
||||
volOptions.VolName = volName
|
||||
// Mapping RBD image
|
||||
devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets())
|
||||
devicePath, err := attachRBDImage(volOptions, volOptions.UserId, req.GetNodePublishSecrets())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -31,11 +31,9 @@ import (
|
||||
|
||||
// PluginFolder defines the location of rbdplugin
|
||||
const (
|
||||
PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin"
|
||||
// RBDUserID used as a key in credentials map to extract the key which is
|
||||
// passed be the provisioner, the value od RBDUserID must match to the key used
|
||||
// in Secret object.
|
||||
RBDUserID = "admin"
|
||||
PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin"
|
||||
rbdDefaultAdminId = "admin"
|
||||
rbdDefaultUserId = rbdDefaultAdminId
|
||||
)
|
||||
|
||||
type rbd struct {
|
||||
@ -51,25 +49,66 @@ type rbd struct {
|
||||
|
||||
var (
|
||||
rbdDriver *rbd
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
)
|
||||
|
||||
var rbdVolumes map[string]rbdVolume
|
||||
var rbdVolumes map[string]*rbdVolume
|
||||
var rbdSnapshots map[string]*rbdSnapshot
|
||||
|
||||
// Init checks for the persistent volume file and loads all found volumes
|
||||
// into a memory structure
|
||||
func init() {
|
||||
rbdVolumes = map[string]rbdVolume{}
|
||||
rbdVolumes = map[string]*rbdVolume{}
|
||||
rbdSnapshots = map[string]*rbdSnapshot{}
|
||||
if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) {
|
||||
glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller"))
|
||||
if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil {
|
||||
glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err)
|
||||
}
|
||||
} else {
|
||||
// Since "controller" folder exists, it means the rbdplugin has already been running, it means
|
||||
// there might be some volumes left, they must be re-inserted into rbdVolumes map
|
||||
loadExVolumes()
|
||||
}
|
||||
if _, err := os.Stat(path.Join(PluginFolder, "controller-snap")); os.IsNotExist(err) {
|
||||
glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller-snap"))
|
||||
if err := os.Mkdir(path.Join(PluginFolder, "controller-snap"), 0755); err != nil {
|
||||
glog.Fatalf("Failed to create a controller's snapshots folder with error: %v\n", err)
|
||||
}
|
||||
} else {
|
||||
// Since "controller-snap" folder exists, it means the rbdplugin has already been running, it means
|
||||
// there might be some snapshots left, they must be re-inserted into rbdSnapshots map
|
||||
loadExSnapshots()
|
||||
}
|
||||
}
|
||||
|
||||
// loadExSnapshots check for any *.json files in the PluginFolder/controller-snap folder
|
||||
// and loads then into rbdSnapshots map
|
||||
func loadExSnapshots() {
|
||||
rbdSnap := rbdSnapshot{}
|
||||
files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller-snap"))
|
||||
if err != nil {
|
||||
glog.Infof("rbd: failed to read controller's snapshots folder: %s error:%v", path.Join(PluginFolder, "controller-snap"), err)
|
||||
return
|
||||
}
|
||||
// Since "controller" folder exists, it means the rbdplugin has already been running, it means
|
||||
// there might be some volumes left, they must be re-inserted into rbdVolumes map
|
||||
loadExVolumes()
|
||||
for _, f := range files {
|
||||
if !strings.HasSuffix(f.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
fp, err := os.Open(path.Join(PluginFolder, "controller-snap", f.Name()))
|
||||
if err != nil {
|
||||
glog.Infof("rbd: open file: %s err %%v", f.Name(), err)
|
||||
continue
|
||||
}
|
||||
decoder := json.NewDecoder(fp)
|
||||
if err = decoder.Decode(&rbdSnap); err != nil {
|
||||
glog.Infof("rbd: decode file: %s err: %v", f.Name(), err)
|
||||
fp.Close()
|
||||
continue
|
||||
}
|
||||
rbdSnapshots[rbdSnap.SnapID] = &rbdSnap
|
||||
}
|
||||
glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap"))
|
||||
}
|
||||
|
||||
// loadExVolumes check for any *.json files in the PluginFolder/controller folder
|
||||
@ -78,7 +117,7 @@ func loadExVolumes() {
|
||||
rbdVol := rbdVolume{}
|
||||
files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller"))
|
||||
if err != nil {
|
||||
glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
|
||||
glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
|
||||
return
|
||||
}
|
||||
for _, f := range files {
|
||||
@ -96,7 +135,7 @@ func loadExVolumes() {
|
||||
fp.Close()
|
||||
continue
|
||||
}
|
||||
rbdVolumes[rbdVol.VolID] = rbdVol
|
||||
rbdVolumes[rbdVol.VolID] = &rbdVol
|
||||
}
|
||||
glog.Infof("rbd: Loaded %d volumes from %s", len(rbdVolumes), path.Join(PluginFolder, "controller"))
|
||||
}
|
||||
@ -134,6 +173,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
|
||||
rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
|
||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
||||
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
|
||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
|
||||
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
|
||||
})
|
||||
rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})
|
||||
|
||||
|
||||
@ -54,6 +54,21 @@ type rbdVolume struct {
|
||||
ImageFormat string `json:"imageFormat"`
|
||||
ImageFeatures string `json:"imageFeatures"`
|
||||
VolSize int64 `json:"volSize"`
|
||||
AdminId string `json:"adminId"`
|
||||
UserId string `json:"userId"`
|
||||
}
|
||||
|
||||
type rbdSnapshot struct {
|
||||
SourceVolumeID string `json:"sourceVolumeID"`
|
||||
VolName string `json:"volName"`
|
||||
SnapName string `json:"snapName"`
|
||||
SnapID string `json:"sanpID"`
|
||||
Monitors string `json:"monitors"`
|
||||
Pool string `json:"pool"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
AdminId string `json:"adminId"`
|
||||
UserId string `json:"userId"`
|
||||
}
|
||||
|
||||
var (
|
||||
@ -70,7 +85,7 @@ func getRBDKey(id string, credentials map[string]string) (string, error) {
|
||||
}
|
||||
|
||||
// CreateImage creates a new ceph image with provision and volume options.
|
||||
func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error {
|
||||
func createRBDImage(pOpts *rbdVolume, volSz int, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
@ -79,16 +94,16 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string)
|
||||
image := pOpts.VolName
|
||||
volSzGB := fmt.Sprintf("%dG", volSz)
|
||||
|
||||
key, err := getRBDKey(RBDUserID, credentials)
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key)
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, adminId, key)
|
||||
} else {
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key)
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, adminId, key)
|
||||
}
|
||||
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat}
|
||||
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", adminId, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat}
|
||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||
args = append(args, "--image-feature", pOpts.ImageFeatures)
|
||||
}
|
||||
@ -103,20 +118,21 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string)
|
||||
|
||||
// rbdStatus checks if there is watcher on the image.
|
||||
// It returns true if there is a watcher onthe image, otherwise returns false.
|
||||
func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) {
|
||||
func rbdStatus(pOpts *rbdVolume, userId string, credentials map[string]string) (bool, string, error) {
|
||||
var err error
|
||||
var output string
|
||||
var cmd []byte
|
||||
|
||||
image := pOpts.VolName
|
||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||
key, err := getRBDKey(RBDUserID, credentials)
|
||||
|
||||
key, err := getRBDKey(userId, credentials)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
|
||||
args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key}
|
||||
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, userId, key)
|
||||
args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userId, "--key=" + key}
|
||||
cmd, err = execCommand("rbd", args)
|
||||
output = string(cmd)
|
||||
|
||||
@ -143,10 +159,10 @@ func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, e
|
||||
}
|
||||
|
||||
// DeleteImage deletes a ceph image with provision and volume options.
|
||||
func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error {
|
||||
func deleteRBDImage(pOpts *rbdVolume, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
image := pOpts.VolName
|
||||
found, _, err := rbdStatus(pOpts, credentials)
|
||||
found, _, err := rbdStatus(pOpts, adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -154,13 +170,13 @@ func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error {
|
||||
glog.Info("rbd is still being used ", image)
|
||||
return fmt.Errorf("rbd %s is still being used", image)
|
||||
}
|
||||
key, err := getRBDKey(RBDUserID, credentials)
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
|
||||
args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key}
|
||||
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key)
|
||||
args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminId, "-m", pOpts.Monitors, "--key=" + key}
|
||||
output, err = execCommand("rbd", args)
|
||||
if err == nil {
|
||||
return nil
|
||||
@ -204,11 +220,51 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rbdVol.AdminId, ok = volOptions["adminid"]
|
||||
if !ok {
|
||||
rbdVol.AdminId = rbdDefaultAdminId
|
||||
}
|
||||
rbdVol.UserId, ok = volOptions["userid"]
|
||||
if !ok {
|
||||
rbdVol.UserId = rbdDefaultUserId
|
||||
}
|
||||
return rbdVol, nil
|
||||
}
|
||||
|
||||
func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) {
|
||||
func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) {
|
||||
var ok bool
|
||||
rbdSnap := &rbdSnapshot{}
|
||||
rbdSnap.Pool, ok = snapOptions["pool"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter pool")
|
||||
}
|
||||
rbdSnap.Monitors, ok = snapOptions["monitors"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter monitors")
|
||||
}
|
||||
rbdSnap.AdminId, ok = snapOptions["adminid"]
|
||||
if !ok {
|
||||
rbdSnap.AdminId = rbdDefaultAdminId
|
||||
}
|
||||
rbdSnap.UserId, ok = snapOptions["userid"]
|
||||
if !ok {
|
||||
rbdSnap.UserId = rbdDefaultUserId
|
||||
}
|
||||
|
||||
return rbdSnap, nil
|
||||
}
|
||||
|
||||
func hasSnapshotFeature(imageFeatures string) bool {
|
||||
arr := strings.Split(imageFeatures, ",")
|
||||
for _, f := range arr {
|
||||
if f == "layering" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) {
|
||||
var err error
|
||||
var output []byte
|
||||
|
||||
@ -229,7 +285,7 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin
|
||||
Steps: rbdImageWatcherSteps,
|
||||
}
|
||||
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
|
||||
used, rbdOutput, err := rbdStatus(volOptions, credentials)
|
||||
used, rbdOutput, err := rbdStatus(volOptions, userId, credentials)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
|
||||
}
|
||||
@ -245,13 +301,13 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin
|
||||
}
|
||||
|
||||
glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
|
||||
key, err := getRBDKey(RBDUserID, credentials)
|
||||
key, err := getRBDKey(userId, credentials)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
output, err = execCommand("rbd", []string{
|
||||
"map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key})
|
||||
"map", image, "--pool", volOptions.Pool, "--id", userId, "-m", volOptions.Monitors, "--key=" + key})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
|
||||
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
|
||||
@ -376,18 +432,191 @@ func deleteVolInfo(image string, persistentStoragePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getRBDVolumeByID(volumeID string) (rbdVolume, error) {
|
||||
func persistSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error {
|
||||
file := path.Join(persistentStoragePath, snapshot+".json")
|
||||
fp, err := os.Create(file)
|
||||
if err != nil {
|
||||
glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err)
|
||||
return fmt.Errorf("rbd: create err %s/%s", file, err)
|
||||
}
|
||||
defer fp.Close()
|
||||
encoder := json.NewEncoder(fp)
|
||||
if err = encoder.Encode(snapInfo); err != nil {
|
||||
glog.Errorf("rbd: failed to encode snapInfo: %+v for file: %s with error: %v\n", snapInfo, file, err)
|
||||
return fmt.Errorf("rbd: encode err: %v", err)
|
||||
}
|
||||
glog.Infof("rbd: successfully saved snapInfo: %+v into file: %s\n", snapInfo, file)
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error {
|
||||
file := path.Join(persistentStoragePath, snapshot+".json")
|
||||
fp, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rbd: open err %s/%s", file, err)
|
||||
}
|
||||
defer fp.Close()
|
||||
|
||||
decoder := json.NewDecoder(fp)
|
||||
if err = decoder.Decode(snapInfo); err != nil {
|
||||
return fmt.Errorf("rbd: decode err: %v.", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteSnapInfo(snapshot string, persistentStoragePath string) error {
|
||||
file := path.Join(persistentStoragePath, snapshot+".json")
|
||||
glog.Infof("rbd: Deleting file for Snapshot: %s at: %s resulting path: %+v\n", snapshot, persistentStoragePath, file)
|
||||
err := os.Remove(file)
|
||||
if err != nil {
|
||||
if err != os.ErrNotExist {
|
||||
return fmt.Errorf("rbd: error removing file: %s/%s", file, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getRBDVolumeByID(volumeID string) (*rbdVolume, error) {
|
||||
if rbdVol, ok := rbdVolumes[volumeID]; ok {
|
||||
return rbdVol, nil
|
||||
}
|
||||
return rbdVolume{}, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID)
|
||||
return nil, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID)
|
||||
}
|
||||
|
||||
func getRBDVolumeByName(volName string) (rbdVolume, error) {
|
||||
func getRBDVolumeByName(volName string) (*rbdVolume, error) {
|
||||
for _, rbdVol := range rbdVolumes {
|
||||
if rbdVol.VolName == volName {
|
||||
return rbdVol, nil
|
||||
}
|
||||
}
|
||||
return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName)
|
||||
return nil, fmt.Errorf("volume name %s does not exit in the volumes list", volName)
|
||||
}
|
||||
|
||||
func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) {
|
||||
for _, rbdSnap := range rbdSnapshots {
|
||||
if rbdSnap.SnapName == snapName {
|
||||
return rbdSnap, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName)
|
||||
}
|
||||
|
||||
func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
mon := pOpts.Monitors
|
||||
image := pOpts.VolName
|
||||
snapID := pOpts.SnapID
|
||||
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key)
|
||||
args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key}
|
||||
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to protect snapshot: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
mon := pOpts.Monitors
|
||||
image := pOpts.VolName
|
||||
snapID := pOpts.SnapID
|
||||
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key)
|
||||
args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key}
|
||||
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create snapshot: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
mon := pOpts.Monitors
|
||||
image := pOpts.VolName
|
||||
snapID := pOpts.SnapID
|
||||
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key)
|
||||
args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key}
|
||||
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unprotect snapshot: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
mon := pOpts.Monitors
|
||||
image := pOpts.VolName
|
||||
snapID := pOpts.SnapID
|
||||
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key)
|
||||
args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key}
|
||||
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete snapshot: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminId string, credentials map[string]string) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
mon := pVolOpts.Monitors
|
||||
image := pVolOpts.VolName
|
||||
snapID := pSnapOpts.SnapID
|
||||
|
||||
key, err := getRBDKey(adminId, credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, adminId, key)
|
||||
args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminId, "-m", mon, "--key=" + key}
|
||||
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to restore snapshot: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user