Merge pull request #136 from rootfs/cephfs-mon

allow ceph mon stored in secret so when mon changes, cephfs driver can get latest mons and override old ones

docs/deploy-cephfs.md

@@ -38,6 +38,7 @@ Option | Default value | Description
 Parameter | Required | Description
 --------- | -------- | -----------
 `monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
+`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. If both `monitors` and `monValueFromSecret` are set and the monitors set in the secret exists, `monValueFromSecret` takes precedence. 
 `mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
 `provisionVolume` | yes | Mode of operation. BOOL value. If `true`, a new CephFS volume will be provisioned. If `false`, an existing volume will be used.
 `pool` | for `provisionVolume=true` | Ceph pool into which the volume shall be created

pkg/cephfs/controllerserver.go

@@ -44,14 +44,14 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}
 	// Configuration
-	volOptions, err := newVolumeOptions(req.GetParameters())
+	secret := req.GetSecrets()
+	volOptions, err := newVolumeOptions(req.GetParameters(), secret)
 	if err != nil {
 		glog.Errorf("validation of volume options failed: %v", err)
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
 	volId := makeVolumeID(req.GetName())
-
 	conf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
 	if err = conf.writeToFile(); err != nil {
 		glog.Errorf("failed to write ceph config file to %s: %v", getCephConfPath(volId), err)
@@ -62,7 +62,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 
 	if volOptions.ProvisionVolume {
 		// Admin credentials are required
-		cr, err := getAdminCredentials(req.GetSecrets())
+		cr, err := getAdminCredentials(secret)
 		if err != nil {
 			return nil, status.Error(codes.InvalidArgument, err.Error())
 		}
@@ -124,10 +124,17 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		glog.Warningf("volume %s is provisioned statically, aborting delete", volId)
 		return &csi.DeleteVolumeResponse{}, nil
 	}
+	// mons may have changed since create volume,
+	// retrieve the latest mons and override old mons
+	secret := req.GetSecrets()
+	if mon, err := getMonValFromSecret(secret); err == nil && len(mon) > 0 {
+		glog.Infof("override old mons [%q] with [%q]", ce.VolOptions.Monitors, mon)
+		ce.VolOptions.Monitors = mon
+	}
 
 	// Deleting a volume requires admin credentials
 
-	cr, err := getAdminCredentials(req.GetSecrets())
+	cr, err := getAdminCredentials(secret)
 	if err != nil {
 		glog.Errorf("failed to retrieve admin credentials: %v", err)
 		return nil, status.Error(codes.InvalidArgument, err.Error())

pkg/cephfs/credentials.go

@@ -23,6 +23,7 @@ const (
 	credUserKey  = "userKey"
 	credAdminId  = "adminID"
 	credAdminKey = "adminKey"
+	credMonitors = "monitors"
 )
 
 type credentials struct {
@@ -54,3 +55,10 @@ func getUserCredentials(secrets map[string]string) (*credentials, error) {
 func getAdminCredentials(secrets map[string]string) (*credentials, error) {
 	return getCredentials(credAdminId, credAdminKey, secrets)
 }
+
+func getMonValFromSecret(secrets map[string]string) (string, error) {
+	if mons, ok := secrets[credMonitors]; ok {
+		return mons, nil
+	}
+	return "", fmt.Errorf("missing %q", credMonitors)
+}

pkg/cephfs/nodeserver.go

@@ -38,13 +38,13 @@ func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi
 		userCr *credentials
 		err    error
 	)
-
+	secret := req.GetSecrets()
 	if volOptions.ProvisionVolume {
 		// The volume is provisioned dynamically, get the credentials directly from Ceph
 
 		// First, store admin credentials - those are needed for retrieving the user credentials
 
-		adminCr, err := getAdminCredentials(req.GetSecrets())
+		adminCr, err := getAdminCredentials(secret)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
 		}
@@ -64,7 +64,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi
 	} else {
 		// The volume is pre-made, credentials are in node stage secrets
 
-		userCr, err = getUserCredentials(req.GetSecrets())
+		userCr, err = getUserCredentials(secret)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
 		}
@@ -87,7 +87,8 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	stagingTargetPath := req.GetStagingTargetPath()
 	volId := volumeID(req.GetVolumeId())
 
-	volOptions, err := newVolumeOptions(req.GetVolumeContext())
+	secret := req.GetSecrets()
+	volOptions, err := newVolumeOptions(req.GetVolumeContext(), secret)
 	if err != nil {
 		glog.Errorf("error reading volume options for volume %s: %v", volId, err)
 		return nil, status.Error(codes.InvalidArgument, err.Error())

pkg/cephfs/volumeoptions.go

@@ -28,6 +28,8 @@ type volumeOptions struct {
 
 	Mounter         string `json:"mounter"`
 	ProvisionVolume bool   `json:"provisionVolume"`
+
+	MonValueFromSecret string `json:"monValueFromSecret"`
 }
 
 func validateNonEmptyField(field, fieldName string) error {
@@ -40,7 +42,9 @@ func validateNonEmptyField(field, fieldName string) error {
 
 func (o *volumeOptions) validate() error {
 	if err := validateNonEmptyField(o.Monitors, "monitors"); err != nil {
-		return err
+		if err = validateNonEmptyField(o.MonValueFromSecret, "monValueFromSecret"); err != nil {
+			return err
+		}
 	}
 
 	if err := validateNonEmptyField(o.RootPath, "rootPath"); err != nil {
@@ -89,17 +93,25 @@ func validateMounter(m string) error {
 	return nil
 }
 
-func newVolumeOptions(volOptions map[string]string) (*volumeOptions, error) {
+func newVolumeOptions(volOptions, secret map[string]string) (*volumeOptions, error) {
 	var (
 		opts                volumeOptions
 		provisionVolumeBool string
 		err                 error
 	)
 
-	if err = extractOption(&opts.Monitors, "monitors", volOptions); err != nil {
+	// extract mon from secret first
-		return nil, err
+	if err = extractOption(&opts.MonValueFromSecret, "monValueFromSecret", volOptions); err == nil {
+		if mon, err := getMonValFromSecret(secret); err == nil && len(mon) > 0 {
+			opts.Monitors = mon
+		}
+	}
+	if len(opts.Monitors) == 0 {
+		// if not set in secret, get it from parameter
+		if err = extractOption(&opts.Monitors, "monitors", volOptions); err != nil {
+			return nil, fmt.Errorf("either monitors or monValueFromSecret should be set")
+		}
 	}
-
 	if err = extractOption(&provisionVolumeBool, "provisionVolume", volOptions); err != nil {
 		return nil, err
 	}