allow ceph mon stored in secret so when mon changes, cephfs driver can get latest mons and override old ones

Signed-off-by: Huamin Chen <hchen@redhat.com>

pkg/cephfs/controllerserver.go

@@ -51,7 +51,12 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	}
 
 	volId := makeVolumeID(req.GetName())
-
+	secret := req.GetSecrets()
+	if len(volOptions.Monitors) == 0 {
+		if mon, err := getMonValFromSecret(secret); err == nil && len(mon) > 0 {
+			volOptions.Monitors = mon
+		}
+	}
 	conf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
 	if err = conf.writeToFile(); err != nil {
 		glog.Errorf("failed to write ceph config file to %s: %v", getCephConfPath(volId), err)
@@ -62,7 +67,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 
 	if volOptions.ProvisionVolume {
 		// Admin credentials are required
-		cr, err := getAdminCredentials(req.GetSecrets())
+		cr, err := getAdminCredentials(secret)
 		if err != nil {
 			return nil, status.Error(codes.InvalidArgument, err.Error())
 		}
@@ -124,10 +129,17 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		glog.Warningf("volume %s is provisioned statically, aborting delete", volId)
 		return &csi.DeleteVolumeResponse{}, nil
 	}
+	// mons may have changed since create volume,
+	// retrieve the latest mons and override old mons
+	secret := req.GetSecrets()
+	if mon, err := getMonValFromSecret(secret); err == nil && len(mon) > 0 {
+		glog.Infof("override old mons [%q] with [%q]", ce.VolOptions.Monitors, mon)
+		ce.VolOptions.Monitors = mon
+	}
 
 	// Deleting a volume requires admin credentials
 
-	cr, err := getAdminCredentials(req.GetSecrets())
+	cr, err := getAdminCredentials(secret)
 	if err != nil {
 		glog.Errorf("failed to retrieve admin credentials: %v", err)
 		return nil, status.Error(codes.InvalidArgument, err.Error())

pkg/cephfs/credentials.go

@@ -23,6 +23,7 @@ const (
 	credUserKey  = "userKey"
 	credAdminId  = "adminID"
 	credAdminKey = "adminKey"
+	credMonitors = "monitors"
 )
 
 type credentials struct {
@@ -54,3 +55,10 @@ func getUserCredentials(secrets map[string]string) (*credentials, error) {
 func getAdminCredentials(secrets map[string]string) (*credentials, error) {
 	return getCredentials(credAdminId, credAdminKey, secrets)
 }
+
+func getMonValFromSecret(secrets map[string]string) (string, error) {
+	if mons, ok := secrets[credMonitors]; ok {
+		return mons, nil
+	}
+	return "", fmt.Errorf("missing %q", credMonitors)
+}

pkg/cephfs/nodeserver.go

@@ -38,13 +38,13 @@ func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi
 		userCr *credentials
 		err    error
 	)
-
+	secret := req.GetSecrets()
 	if volOptions.ProvisionVolume {
 		// The volume is provisioned dynamically, get the credentials directly from Ceph
 
 		// First, store admin credentials - those are needed for retrieving the user credentials
 
-		adminCr, err := getAdminCredentials(req.GetSecrets())
+		adminCr, err := getAdminCredentials(secret)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
 		}
@@ -64,7 +64,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi
 	} else {
 		// The volume is pre-made, credentials are in node stage secrets
 
-		userCr, err = getUserCredentials(req.GetSecrets())
+		userCr, err = getUserCredentials(secret)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
 		}
@@ -103,6 +103,13 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	// mons may have changed since create volume,
+	// retrieve the latest mons and override old mons
+	secret := req.GetSecrets()
+	if mon, err := getMonValFromSecret(secret); err == nil && len(mon) > 0 {
+		glog.Infof("override old mons [%q] with [%q]", volOptions.Monitors, mon)
+		volOptions.Monitors = mon
+	}
 	cephConf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
 	if err = cephConf.writeToFile(); err != nil {
 		glog.Errorf("failed to write ceph config file to %s for volume %s: %v", getCephConfPath(volId), volId, err)

pkg/cephfs/volumeoptions.go

@@ -28,6 +28,8 @@ type volumeOptions struct {
 
 	Mounter         string `json:"mounter"`
 	ProvisionVolume bool   `json:"provisionVolume"`
+
+	MonValueFromSecret string `json:"monValueFromSecret"`
 }
 
 func validateNonEmptyField(field, fieldName string) error {
@@ -40,7 +42,9 @@ func validateNonEmptyField(field, fieldName string) error {
 
 func (o *volumeOptions) validate() error {
 	if err := validateNonEmptyField(o.Monitors, "monitors"); err != nil {
-		return err
+		if err = validateNonEmptyField(o.MonValueFromSecret, "monValueFromSecret"); err != nil {
+			return err
+		}
 	}
 
 	if err := validateNonEmptyField(o.RootPath, "rootPath"); err != nil {
@@ -97,7 +101,9 @@ func newVolumeOptions(volOptions map[string]string) (*volumeOptions, error) {
 	)
 
 	if err = extractOption(&opts.Monitors, "monitors", volOptions); err != nil {
-		return nil, err
+		if err = extractOption(&opts.MonValueFromSecret, "monValueFromSecret", volOptions); err != nil {
+			return nil, err
+		}
 	}
 
 	if err = extractOption(&provisionVolumeBool, "provisionVolume", volOptions); err != nil {