From c93466b00965d832d8fc19c6ef28e9d77aee54c4 Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Wed, 8 Aug 2018 14:41:45 +0900 Subject: [PATCH 1/7] Implement NodeGetInfo for csi spec 3.0 --- pkg/cephfs/nodeserver.go | 8 ++++++++ pkg/rbd/nodeserver.go | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index d66b6a89a..6280b6d0f 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -212,3 +212,11 @@ func (ns *nodeServer) NodeUnstageVolume( *csi.NodeUnstageVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } + +func (ns *nodeServer) NodeGetInfo( + ctx context.Context, + req *csi.NodeGetInfoRequest) ( + *csi.NodeGetInfoResponse, error) { + + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 7091ca99c..b221b00ab 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -146,3 +146,11 @@ func (ns *nodeServer) NodeUnstageVolume( return nil, status.Error(codes.Unimplemented, "") } + +func (ns *nodeServer) NodeGetInfo( + ctx context.Context, + req *csi.NodeGetInfoRequest) ( + *csi.NodeGetInfoResponse, error) { + + return nil, status.Error(codes.Unimplemented, "") +} From b1ccdbb1547ef3b29b240a596c82683f109238b2 Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Wed, 8 Aug 2018 14:42:17 +0900 Subject: [PATCH 2/7] Support snapshot feature in rbdplugin --- pkg/rbd/controllerserver.go | 241 +++++++++++++++++++++++++++++++++++- pkg/rbd/rbd.go | 53 +++++++- pkg/rbd/rbd_util.go | 199 +++++++++++++++++++++++++++++ 3 files changed, 483 insertions(+), 10 deletions(-) diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 4c986e776..546a27d90 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -18,7 +18,10 @@ package rbd import ( "fmt" + "os/exec" "path" + "syscall" + "time" "github.com/container-storage-interface/spec/lib/go/csi/v0" "github.com/golang/glog" @@ -77,7 +80,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - // Generating Volume Name and Volume ID, as accoeding to CSI spec they MUST be different + // Generating Volume Name and Volume ID, as according to CSI spec they MUST be different volName := req.GetName() uniqueID := uuid.NewUUID().String() if len(volName) == 0 { @@ -97,14 +100,39 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // Check if there is already RBD image with requested name found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets()) if !found { - if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { - if err != nil { - glog.Warningf("failed to create volume: %v", err) + // if VolumeContentSource is not nil, this request is for snapshot + if req.VolumeContentSource != nil { + snapshot := req.VolumeContentSource.GetSnapshot() + if snapshot == nil { + return nil, status.Error(codes.InvalidArgument, "Volume Snapshot cannot be empty") + } + + snapshotID := snapshot.GetId() + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty") + } + + rbdSnap := &rbdSnapshot{} + if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { return nil, err } + + err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets()) + if err != nil { + return nil, err + } + glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName) + } else { + if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { + if err != nil { + glog.Warningf("failed to create volume: %v", err) + return nil, err + } + } + glog.V(4).Infof("create volume %s", volName) } - glog.V(4).Infof("create volume %s", volName) } + // Storing volInfo into a persistent file. if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil { glog.Warningf("rbd: failed to store volInfo with error: %v", err) @@ -162,3 +190,206 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req * func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { return &csi.ControllerPublishVolumeResponse{}, nil } + +func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { + glog.Warningf("invalid create snapshot req: %v", req) + return nil, err + } + + // Check sanity of request Snapshot Name, Source Volume Id + if len(req.Name) == 0 { + return nil, status.Error(codes.InvalidArgument, "Snapshot Name cannot be empty") + } + if len(req.SourceVolumeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty") + } + + // Need to check for already existing snapshot name, and if found + // check for the requested source volume id and already allocated source volume id + if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil { + if req.SourceVolumeId == exSnap.SourceVolumeID { + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SizeBytes: exSnap.SizeBytes, + Id: exSnap.SnapID, + SourceVolumeId: exSnap.SourceVolumeID, + CreatedAt: exSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, nil + } + return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Snapshot with the same name: %s but with different source volume id already exist", req.GetName())) + } + + rbdSnap, err := getRBDSnapshotOptions(req.GetParameters()) + if err != nil { + return nil, err + } + + // Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different + snapName := req.GetName() + uniqueID := uuid.NewUUID().String() + rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId()) + if err != nil { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId())) + } + rbdSnap.VolName = rbdVolume.VolName + rbdSnap.SnapName = snapName + snapshotID := "csi-rbd-snapshot-" + uniqueID + rbdSnap.SnapID = snapshotID + rbdSnap.SourceVolumeID = req.GetSourceVolumeId() + rbdSnap.SizeBytes = rbdVolume.VolSize + + err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + // if we already have the snapshot, return the snapshot + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() == int(syscall.EEXIST) { + glog.Warningf("Snapshot with the same name: %s, we return this.", req.GetName()) + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.Warningf("failed to create snapshot: %v", err) + return nil, err + } + } else { + glog.V(4).Infof("create snapshot %s", snapName) + err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + + if err != nil { + err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + if err != nil { + return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) + } + return nil, fmt.Errorf("Snapshot is created but failed to protect snapshot") + } + } + + rbdSnap.CreatedAt = time.Now().UnixNano() + + // Storing snapInfo into a persistent file. + if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { + glog.Warningf("rbd: failed to store sanpInfo with error: %v", err) + return nil, err + } + rbdSnapshots[snapshotID] = *rbdSnap + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: snapshotID, + SourceVolumeId: req.GetSourceVolumeId(), + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, nil +} + +func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { + glog.Warningf("invalid delete snapshot req: %v", req) + return nil, err + } + + snapshotID := req.GetSnapshotId() + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty") + } + rbdSnap := &rbdSnapshot{} + if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { + return nil, err + } + + // Unprotect snapshot + err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()) + if err != nil { + return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } + + // Deleting snapshot + glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) + if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil { + return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } + + // Removing persistent storage file for the unmapped snapshot + if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil { + return nil, err + } + + delete(rbdSnapshots, snapshotID) + + return &csi.DeleteSnapshotResponse{}, nil +} + +func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil { + glog.Warningf("invalid list snapshot req: %v", req) + return nil, err + } + + sourceVolumeId := req.GetSourceVolumeId() + + // TODO (sngchlko) list with token + + // list only a specific snapshot which has snapshot ID + if snapshotID := req.GetSnapshotId(); snapshotID != "" { + if rbdSnap, ok := rbdSnapshots[snapshotID]; ok { + // if source volume ID also set, check source volume id on the cache. + if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID)) + } + return &csi.ListSnapshotsResponse{ + Entries: []*csi.ListSnapshotsResponse_Entry{ + { + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: rbdSnap.SnapID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }, + }, + }, nil + } else { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Snapshot ID %s cannot found", snapshotID)) + } + } + + entries := []*csi.ListSnapshotsResponse_Entry{} + for _, rbdSnap := range rbdSnapshots { + // if source volume ID also set, check source volume id on the cache. + if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + continue + } + entries = append(entries, &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + Id: rbdSnap.SnapID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreatedAt: rbdSnap.CreatedAt, + Status: &csi.SnapshotStatus{ + Type: csi.SnapshotStatus_READY, + }, + }, + }) + } + + return &csi.ListSnapshotsResponse{ + Entries: entries, + }, nil +} diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 569e97219..09395a286 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -51,25 +51,66 @@ type rbd struct { var ( rbdDriver *rbd - version = "0.2.0" + version = "0.3.0" ) var rbdVolumes map[string]rbdVolume +var rbdSnapshots map[string]rbdSnapshot // Init checks for the persistent volume file and loads all found volumes // into a memory structure func init() { rbdVolumes = map[string]rbdVolume{} + rbdSnapshots = map[string]rbdSnapshot{} if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) { glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller")) if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil { glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err) } + } else { + // Since "controller" folder exists, it means the rbdplugin has already been running, it means + // there might be some volumes left, they must be re-inserted into rbdVolumes map + loadExVolumes() + } + if _, err := os.Stat(path.Join(PluginFolder, "controller-snap")); os.IsNotExist(err) { + glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller-snap")) + if err := os.Mkdir(path.Join(PluginFolder, "controller-snap"), 0755); err != nil { + glog.Fatalf("Failed to create a controller's snapshots folder with error: %v\n", err) + } + } else { + // Since "controller-snap" folder exists, it means the rbdplugin has already been running, it means + // there might be some snapshots left, they must be re-inserted into rbdSnapshots map + loadExSnapshots() + } +} + +// loadExSnapshots check for any *.json files in the PluginFolder/controller-snap folder +// and loads then into rbdSnapshots map +func loadExSnapshots() { + rbdSnap := rbdSnapshot{} + files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller-snap")) + if err != nil { + glog.Infof("rbd: failed to read controller's snapshots folder: %s error:%v", path.Join(PluginFolder, "controller-snap"), err) return } - // Since "controller" folder exists, it means the rbdplugin has already been running, it means - // there might be some volumes left, they must be re-inserted into rbdVolumes map - loadExVolumes() + for _, f := range files { + if !strings.HasSuffix(f.Name(), ".json") { + continue + } + fp, err := os.Open(path.Join(PluginFolder, "controller-snap", f.Name())) + if err != nil { + glog.Infof("rbd: open file: %s err %%v", f.Name(), err) + continue + } + decoder := json.NewDecoder(fp) + if err = decoder.Decode(&rbdSnap); err != nil { + glog.Infof("rbd: decode file: %s err: %v", f.Name(), err) + fp.Close() + continue + } + rbdSnapshots[rbdSnap.SnapID] = rbdSnap + } + glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap")) } // loadExVolumes check for any *.json files in the PluginFolder/controller folder @@ -78,7 +119,7 @@ func loadExVolumes() { rbdVol := rbdVolume{} files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller")) if err != nil { - glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err) + glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err) return } for _, f := range files { @@ -134,6 +175,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) { rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, }) rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index afdcd21e5..796ed606a 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -56,6 +56,17 @@ type rbdVolume struct { VolSize int64 `json:"volSize"` } +type rbdSnapshot struct { + SourceVolumeID string `json:"sourceVolumeID"` + VolName string `json:"volName"` + SnapName string `json:"snapName"` + SnapID string `json:"sanpID"` + Monitors string `json:"monitors"` + Pool string `json:"pool"` + CreatedAt int64 `json:"createdAt"` + SizeBytes int64 `json:"sizeBytes"` +} + var ( attachdetachMutex = keymutex.NewKeyMutex() supportedFeatures = sets.NewString("layering") @@ -208,6 +219,21 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { return rbdVol, nil } +func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) { + var ok bool + rbdSnap := &rbdSnapshot{} + rbdSnap.Pool, ok = snapOptions["pool"] + if !ok { + return nil, fmt.Errorf("Missing required parameter pool") + } + rbdSnap.Monitors, ok = snapOptions["monitors"] + if !ok { + return nil, fmt.Errorf("Missing required parameter monitors") + } + + return rbdSnap, nil +} + func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) { var err error var output []byte @@ -376,6 +402,50 @@ func deleteVolInfo(image string, persistentStoragePath string) error { return nil } +func persistSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error { + file := path.Join(persistentStoragePath, snapshot+".json") + fp, err := os.Create(file) + if err != nil { + glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err) + return fmt.Errorf("rbd: create err %s/%s", file, err) + } + defer fp.Close() + encoder := json.NewEncoder(fp) + if err = encoder.Encode(snapInfo); err != nil { + glog.Errorf("rbd: failed to encode snapInfo: %+v for file: %s with error: %v\n", snapInfo, file, err) + return fmt.Errorf("rbd: encode err: %v", err) + } + glog.Infof("rbd: successfully saved snapInfo: %+v into file: %s\n", snapInfo, file) + return nil +} + +func loadSnapInfo(snapshot string, persistentStoragePath string, snapInfo *rbdSnapshot) error { + file := path.Join(persistentStoragePath, snapshot+".json") + fp, err := os.Open(file) + if err != nil { + return fmt.Errorf("rbd: open err %s/%s", file, err) + } + defer fp.Close() + + decoder := json.NewDecoder(fp) + if err = decoder.Decode(snapInfo); err != nil { + return fmt.Errorf("rbd: decode err: %v.", err) + } + return nil +} + +func deleteSnapInfo(snapshot string, persistentStoragePath string) error { + file := path.Join(persistentStoragePath, snapshot+".json") + glog.Infof("rbd: Deleting file for Snapshot: %s at: %s resulting path: %+v\n", snapshot, persistentStoragePath, file) + err := os.Remove(file) + if err != nil { + if err != os.ErrNotExist { + return fmt.Errorf("rbd: error removing file: %s/%s", file, err) + } + } + return nil +} + func getRBDVolumeByID(volumeID string) (rbdVolume, error) { if rbdVol, ok := rbdVolumes[volumeID]; ok { return rbdVol, nil @@ -391,3 +461,132 @@ func getRBDVolumeByName(volName string) (rbdVolume, error) { } return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName) } + +func getRBDSnapshotByName(snapName string) (rbdSnapshot, error) { + for _, rbdSnap := range rbdSnapshots { + if rbdSnap.SnapName == snapName { + return rbdSnap, nil + } + } + return rbdSnapshot{}, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) +} + +func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to protect snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to create snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to unprotect snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pOpts.Monitors + image := pOpts.VolName + snapID := pOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to delete snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} + +func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials map[string]string) error { + var output []byte + var err error + + mon := pVolOpts.Monitors + image := pVolOpts.VolName + snapID := pSnapOpts.SnapID + + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err + } + glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, RBDUserID, key) + args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", RBDUserID, "-m", mon, "--key=" + key} + + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to restore snapshot: %v, command output: %s", err, string(output)) + } + + return nil +} From f0fba1240ad11c3446e381391fbedc9ead783dcf Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Wed, 8 Aug 2018 20:22:23 +0900 Subject: [PATCH 3/7] Revert "Implement NodeGetInfo for csi spec 3.0" This reverts commit c93466b00965d832d8fc19c6ef28e9d77aee54c4. --- pkg/cephfs/nodeserver.go | 8 -------- pkg/rbd/nodeserver.go | 8 -------- 2 files changed, 16 deletions(-) diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index 6280b6d0f..d66b6a89a 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -212,11 +212,3 @@ func (ns *nodeServer) NodeUnstageVolume( *csi.NodeUnstageVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } - -func (ns *nodeServer) NodeGetInfo( - ctx context.Context, - req *csi.NodeGetInfoRequest) ( - *csi.NodeGetInfoResponse, error) { - - return nil, status.Error(codes.Unimplemented, "") -} diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index b221b00ab..7091ca99c 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -146,11 +146,3 @@ func (ns *nodeServer) NodeUnstageVolume( return nil, status.Error(codes.Unimplemented, "") } - -func (ns *nodeServer) NodeGetInfo( - ctx context.Context, - req *csi.NodeGetInfoRequest) ( - *csi.NodeGetInfoResponse, error) { - - return nil, status.Error(codes.Unimplemented, "") -} From 7d90783f0349e8a89fefd6b1e4c4c8932bb63219 Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Thu, 9 Aug 2018 22:06:51 +0900 Subject: [PATCH 4/7] fix nit --- pkg/rbd/controllerserver.go | 17 +++++++++-------- pkg/rbd/rbd.go | 12 ++++++------ pkg/rbd/rbd_util.go | 12 ++++++------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 546a27d90..9dbe1f55a 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -108,7 +108,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } snapshotID := snapshot.GetId() - if snapshotID == "" { + if len(snapshotID) == 0 { return nil, status.Error(codes.InvalidArgument, "Volume Snapshot ID cannot be empty") } @@ -137,7 +137,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil { glog.Warningf("rbd: failed to store volInfo with error: %v", err) } - rbdVolumes[volumeID] = *rbdVol + rbdVolumes[volumeID] = rbdVol return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ Id: volumeID, @@ -238,7 +238,7 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS } rbdSnap.VolName = rbdVolume.VolName rbdSnap.SnapName = snapName - snapshotID := "csi-rbd-snapshot-" + uniqueID + snapshotID := "csi-rbd-" + rbdVolume.VolName + "-snap-" + uniqueID rbdSnap.SnapID = snapshotID rbdSnap.SourceVolumeID = req.GetSourceVolumeId() rbdSnap.SizeBytes = rbdVolume.VolSize @@ -280,9 +280,10 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS // Storing snapInfo into a persistent file. if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { glog.Warningf("rbd: failed to store sanpInfo with error: %v", err) + return nil, err } - rbdSnapshots[snapshotID] = *rbdSnap + rbdSnapshots[snapshotID] = rbdSnap return &csi.CreateSnapshotResponse{ Snapshot: &csi.Snapshot{ SizeBytes: rbdSnap.SizeBytes, @@ -303,7 +304,7 @@ func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS } snapshotID := req.GetSnapshotId() - if snapshotID == "" { + if len(snapshotID) == 0 { return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty") } rbdSnap := &rbdSnapshot{} @@ -344,10 +345,10 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap // TODO (sngchlko) list with token // list only a specific snapshot which has snapshot ID - if snapshotID := req.GetSnapshotId(); snapshotID != "" { + if snapshotID := req.GetSnapshotId(); len(snapshotID) != 0 { if rbdSnap, ok := rbdSnapshots[snapshotID]; ok { // if source volume ID also set, check source volume id on the cache. - if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + if len(sourceVolumeId) != 0 && rbdSnap.SourceVolumeID != sourceVolumeId { return nil, status.Error(codes.Unknown, fmt.Sprintf("Requested Source Volume ID %s is different from %s", sourceVolumeId, rbdSnap.SourceVolumeID)) } return &csi.ListSnapshotsResponse{ @@ -373,7 +374,7 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap entries := []*csi.ListSnapshotsResponse_Entry{} for _, rbdSnap := range rbdSnapshots { // if source volume ID also set, check source volume id on the cache. - if sourceVolumeId != "" && rbdSnap.SourceVolumeID != sourceVolumeId { + if len(sourceVolumeId) != 0 && rbdSnap.SourceVolumeID != sourceVolumeId { continue } entries = append(entries, &csi.ListSnapshotsResponse_Entry{ diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 09395a286..d5c6d000e 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -54,14 +54,14 @@ var ( version = "0.3.0" ) -var rbdVolumes map[string]rbdVolume -var rbdSnapshots map[string]rbdSnapshot +var rbdVolumes map[string]*rbdVolume +var rbdSnapshots map[string]*rbdSnapshot // Init checks for the persistent volume file and loads all found volumes // into a memory structure func init() { - rbdVolumes = map[string]rbdVolume{} - rbdSnapshots = map[string]rbdSnapshot{} + rbdVolumes = map[string]*rbdVolume{} + rbdSnapshots = map[string]*rbdSnapshot{} if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) { glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller")) if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil { @@ -108,7 +108,7 @@ func loadExSnapshots() { fp.Close() continue } - rbdSnapshots[rbdSnap.SnapID] = rbdSnap + rbdSnapshots[rbdSnap.SnapID] = &rbdSnap } glog.Infof("rbd: Loaded %d snapshots from %s", len(rbdSnapshots), path.Join(PluginFolder, "controller-snap")) } @@ -137,7 +137,7 @@ func loadExVolumes() { fp.Close() continue } - rbdVolumes[rbdVol.VolID] = rbdVol + rbdVolumes[rbdVol.VolID] = &rbdVol } glog.Infof("rbd: Loaded %d volumes from %s", len(rbdVolumes), path.Join(PluginFolder, "controller")) } diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 796ed606a..66c4d1da4 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -446,29 +446,29 @@ func deleteSnapInfo(snapshot string, persistentStoragePath string) error { return nil } -func getRBDVolumeByID(volumeID string) (rbdVolume, error) { +func getRBDVolumeByID(volumeID string) (*rbdVolume, error) { if rbdVol, ok := rbdVolumes[volumeID]; ok { return rbdVol, nil } - return rbdVolume{}, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID) + return nil, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID) } -func getRBDVolumeByName(volName string) (rbdVolume, error) { +func getRBDVolumeByName(volName string) (*rbdVolume, error) { for _, rbdVol := range rbdVolumes { if rbdVol.VolName == volName { return rbdVol, nil } } - return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName) + return nil, fmt.Errorf("volume name %s does not exit in the volumes list", volName) } -func getRBDSnapshotByName(snapName string) (rbdSnapshot, error) { +func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) { for _, rbdSnap := range rbdSnapshots { if rbdSnap.SnapName == snapName { return rbdSnap, nil } } - return rbdSnapshot{}, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) + return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) } func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { From b0e68a52e0e533523cd9bf7e2f06aa894ffeafdc Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Thu, 9 Aug 2018 22:07:00 +0900 Subject: [PATCH 5/7] Refactoring using users --- examples/rbd/secret.yaml | 2 + examples/rbd/storageclass.yaml | 4 ++ pkg/rbd/controllerserver.go | 18 +++---- pkg/rbd/nodeserver.go | 2 +- pkg/rbd/rbd.go | 8 ++- pkg/rbd/rbd_util.go | 98 ++++++++++++++++++++-------------- 6 files changed, 78 insertions(+), 54 deletions(-) diff --git a/examples/rbd/secret.yaml b/examples/rbd/secret.yaml index 594fd8ab4..f15cbcbfd 100644 --- a/examples/rbd/secret.yaml +++ b/examples/rbd/secret.yaml @@ -6,3 +6,5 @@ metadata: data: # Key value corresponds to a user name defined in ceph cluster admin: BASE64-ENCODED-PASSWORD + # Key value corresponds to a user name defined in ceph cluster + kubernetes: BASE64-ENCODED-PASSWORD diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 476c2bef3..e1a8a06ba 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -21,4 +21,8 @@ parameters: csiProvisionerSecretNamespace: default csiNodePublishSecretName: csi-rbd-secret csiNodePublishSecretNamespace: default + + # Ceph users for operating RBD + adminid: admin + userid: kubernetes reclaimPolicy: Delete diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 9dbe1f55a..ba0e307eb 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -98,7 +98,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024) // Check if there is already RBD image with requested name - found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets()) + found, _, _ := rbdStatus(rbdVol, rbdVol.UserId, req.GetControllerCreateSecrets()) if !found { // if VolumeContentSource is not nil, this request is for snapshot if req.VolumeContentSource != nil { @@ -117,13 +117,13 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets()) + err = restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminId, req.GetControllerCreateSecrets()) if err != nil { return nil, err } glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName) } else { - if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { + if err := createRBDImage(rbdVol, volSizeGB, rbdVol.AdminId, req.GetControllerCreateSecrets()); err != nil { if err != nil { glog.Warningf("failed to create volume: %v", err) return nil, err @@ -161,7 +161,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol volName := rbdVol.VolName // Deleting rbd image glog.V(4).Infof("deleting volume %s", volName) - if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil { + if err := deleteRBDImage(rbdVol, rbdVol.AdminId, req.GetControllerDeleteSecrets()); err != nil { glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err) return nil, err } @@ -243,7 +243,7 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS rbdSnap.SourceVolumeID = req.GetSourceVolumeId() rbdSnap.SizeBytes = rbdVolume.VolSize - err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = createSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) // if we already have the snapshot, return the snapshot if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { @@ -264,10 +264,10 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS } } else { glog.V(4).Infof("create snapshot %s", snapName) - err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = protectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) if err != nil { - err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) if err != nil { return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) } @@ -313,14 +313,14 @@ func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS } // Unprotect snapshot - err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()) + err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets()) if err != nil { return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) } // Deleting snapshot glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) - if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil { + if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets()); err != nil { return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 7091ca99c..ca309b18e 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } volOptions.VolName = volName // Mapping RBD image - devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets()) + devicePath, err := attachRBDImage(volOptions, volOptions.UserId, req.GetNodePublishSecrets()) if err != nil { return nil, err } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index d5c6d000e..3f0a04f70 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -31,11 +31,9 @@ import ( // PluginFolder defines the location of rbdplugin const ( - PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" - // RBDUserID used as a key in credentials map to extract the key which is - // passed be the provisioner, the value od RBDUserID must match to the key used - // in Secret object. - RBDUserID = "admin" + PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" + rbdDefaultAdminId = "admin" + rbdDefaultUserId = rbdDefaultAdminId ) type rbd struct { diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 66c4d1da4..7b0c386f6 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -54,6 +54,8 @@ type rbdVolume struct { ImageFormat string `json:"imageFormat"` ImageFeatures string `json:"imageFeatures"` VolSize int64 `json:"volSize"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` } type rbdSnapshot struct { @@ -65,6 +67,8 @@ type rbdSnapshot struct { Pool string `json:"pool"` CreatedAt int64 `json:"createdAt"` SizeBytes int64 `json:"sizeBytes"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` } var ( @@ -81,7 +85,7 @@ func getRBDKey(id string, credentials map[string]string) (string, error) { } // CreateImage creates a new ceph image with provision and volume options. -func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error { +func createRBDImage(pOpts *rbdVolume, volSz int, adminId string, credentials map[string]string) error { var output []byte var err error @@ -90,16 +94,16 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) image := pOpts.VolName volSzGB := fmt.Sprintf("%dG", volSz) - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } if pOpts.ImageFormat == rbdImageFormat2 { - glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key) + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, adminId, key) } else { - glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key) + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, adminId, key) } - args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} + args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", adminId, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { args = append(args, "--image-feature", pOpts.ImageFeatures) } @@ -114,20 +118,21 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) // rbdStatus checks if there is watcher on the image. // It returns true if there is a watcher onthe image, otherwise returns false. -func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) { +func rbdStatus(pOpts *rbdVolume, userId string, credentials map[string]string) (bool, string, error) { var err error var output string var cmd []byte image := pOpts.VolName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - key, err := getRBDKey(RBDUserID, credentials) + + key, err := getRBDKey(userId, credentials) if err != nil { return false, "", err } - glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key} + glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, userId, key) + args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userId, "--key=" + key} cmd, err = execCommand("rbd", args) output = string(cmd) @@ -154,10 +159,10 @@ func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, e } // DeleteImage deletes a ceph image with provision and volume options. -func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error { +func deleteRBDImage(pOpts *rbdVolume, adminId string, credentials map[string]string) error { var output []byte image := pOpts.VolName - found, _, err := rbdStatus(pOpts, credentials) + found, _, err := rbdStatus(pOpts, adminId, credentials) if err != nil { return err } @@ -165,13 +170,13 @@ func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error { glog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key} + glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminId, "-m", pOpts.Monitors, "--key=" + key} output, err = execCommand("rbd", args) if err == nil { return nil @@ -215,7 +220,14 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { } } - + rbdVol.AdminId, ok = volOptions["adminid"] + if !ok { + rbdVol.AdminId = rbdDefaultAdminId + } + rbdVol.UserId, ok = volOptions["userid"] + if !ok { + rbdVol.UserId = rbdDefaultUserId + } return rbdVol, nil } @@ -230,11 +242,19 @@ func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) if !ok { return nil, fmt.Errorf("Missing required parameter monitors") } + rbdSnap.AdminId, ok = snapOptions["adminid"] + if !ok { + rbdSnap.AdminId = rbdDefaultAdminId + } + rbdSnap.UserId, ok = snapOptions["userid"] + if !ok { + rbdSnap.UserId = rbdDefaultUserId + } return rbdSnap, nil } -func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) { +func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) { var err error var output []byte @@ -255,7 +275,7 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin Steps: rbdImageWatcherSteps, } err := wait.ExponentialBackoff(backoff, func() (bool, error) { - used, rbdOutput, err := rbdStatus(volOptions, credentials) + used, rbdOutput, err := rbdStatus(volOptions, userId, credentials) if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } @@ -271,13 +291,13 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin } glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(userId, credentials) if err != nil { return "", err } output, err = execCommand("rbd", []string{ - "map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key}) + "map", image, "--pool", volOptions.Pool, "--id", userId, "-m", volOptions.Monitors, "--key=" + key}) if err != nil { glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) @@ -471,7 +491,7 @@ func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) { return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) } -func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -479,12 +499,12 @@ func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -495,7 +515,7 @@ func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -503,12 +523,12 @@ func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -519,7 +539,7 @@ func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -527,12 +547,12 @@ func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -543,7 +563,7 @@ func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error return nil } -func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -551,12 +571,12 @@ func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -567,7 +587,7 @@ func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials map[string]string) error { +func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -575,12 +595,12 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials ma image := pVolOpts.VolName snapID := pSnapOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, RBDUserID, key) - args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, adminId, key) + args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) From 4312907f7ba6ba1b49d44c69da1aa232422e585d Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Thu, 9 Aug 2018 22:07:06 +0900 Subject: [PATCH 6/7] remove the snapshot if can't store snapshot information --- pkg/rbd/controllerserver.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index ba0e307eb..ca3ce0357 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -279,7 +279,19 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS // Storing snapInfo into a persistent file. if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil { - glog.Warningf("rbd: failed to store sanpInfo with error: %v", err) + glog.Warningf("rbd: failed to store snapInfo with error: %v", err) + + // Unprotect snapshot + err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) + if err != nil { + return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } + + // Deleting snapshot + glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) + if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()); err != nil { + return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) + } return nil, err } From 38aa57592500ec6f23b03b822938705c74a28374 Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Thu, 9 Aug 2018 22:07:13 +0900 Subject: [PATCH 7/7] check snapshot feature --- pkg/rbd/controllerserver.go | 4 ++++ pkg/rbd/rbd_util.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index ca3ce0357..736ec63f9 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -236,6 +236,10 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS if err != nil { return nil, status.Error(codes.NotFound, fmt.Sprintf("Source Volume ID %s cannot found", req.GetSourceVolumeId())) } + if !hasSnapshotFeature(rbdVolume.ImageFeatures) { + return nil, fmt.Errorf("Volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId()) + } + rbdSnap.VolName = rbdVolume.VolName rbdSnap.SnapName = snapName snapshotID := "csi-rbd-" + rbdVolume.VolName + "-snap-" + uniqueID diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 7b0c386f6..e2682b032 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -254,6 +254,16 @@ func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) return rbdSnap, nil } +func hasSnapshotFeature(imageFeatures string) bool { + arr := strings.Split(imageFeatures, ",") + for _, f := range arr { + if f == "layering" { + return true + } + } + return false +} + func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) { var err error var output []byte