1. fix mountcache race conflict

2. support user-defined cache dir
    3. if not define mountcachedir disable mountcache

cmd/cephfs/main.go

@@ -31,6 +31,7 @@ var (
 	nodeID          = flag.String("nodeid", "", "node id")
 	volumeMounter   = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
 	metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
+	mountCacheDir   = flag.String("mountcachedir", "", "mount info cache save dir")
 )
 
 func init() {
@@ -49,6 +50,7 @@ func main() {
 	}
 	//update plugin name
 	cephfs.PluginFolder = cephfs.PluginFolder + *driverName
+	cephfs.MountCacheDir = *mountCacheDir
 
 	cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName)
 	if err != nil {

pkg/cephfs/mountcache.go

@@ -40,10 +40,10 @@ type volumeMountCacheMap struct {
 }
 
 var (
-	csiPersistentVolumeRoot = "/var/lib/kubelet/plugins/kubernetes.io/csi"
+	MountCacheDir          = ""
-	volumeMountCachePrefix  = "cephfs-mount-cache-"
+	volumeMountCachePrefix = "cephfs-mount-cache-"
-	volumeMountCache        volumeMountCacheMap
+	volumeMountCache       volumeMountCacheMap
-	volumeMountCacheMtx     sync.Mutex
+	volumeMountCacheMtx    sync.Mutex
 )
 
 func remountHisMountedPath(name string, v string, nodeID string, cachePersister util.CachePersister) error {
@@ -56,16 +56,18 @@ func remountHisMountedPath(name string, v string, nodeID string, cachePersister
 
 	volumeMountCache.MetadataStore = cachePersister
 
-	volumeMountCache.NodeCacheStore.BasePath = PluginFolder
+	volumeMountCache.NodeCacheStore.BasePath = MountCacheDir
-	volumeMountCache.NodeCacheStore.CacheDir = "volumes-mount-cache"
+	volumeMountCache.NodeCacheStore.CacheDir = ""
 
-	if _, err := os.Stat(csiPersistentVolumeRoot); err != nil {
+	if len(MountCacheDir) == 0 {
-		klog.Infof("mount-cache: csi pv root path %s stat fail %v, may not in daemonset csi plugin, exit", csiPersistentVolumeRoot, err)
+		//if mount cache dir unset, disable remount
-		return err
+		klog.Infof("mount-cache: mountcachedir no define disalbe mount cache.")
+		return nil
 	}
 
+	klog.Infof("mount-cache: MountCacheDir: %s", MountCacheDir)
 	if err := os.MkdirAll(volumeMountCache.NodeCacheStore.BasePath, 0755); err != nil {
-		klog.Fatalf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
+		klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
 		return err
 	}
 	me := &volumeMountEntry{}
@@ -203,6 +205,10 @@ func genVolumeMountCacheFileName(volID string) string {
 }
 
 func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
+	if len(MountCacheDir) == 0 {
+		//if mount cache dir unset, disable remount
+		return nil
+	}
 	volumeMountCacheMtx.Lock()
 	defer volumeMountCacheMtx.Unlock()
 
@@ -210,8 +216,7 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
 	me, ok := volumeMountCache.Volumes[volID]
 	if ok {
 		if me.StagingPath == stagingTargetPath {
-			klog.Infof("mount-cache: node stage volume last cache entry for volume %s stagingTargetPath %s no equal %s",
+			klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID)
-				volID, me.StagingPath, stagingTargetPath)
 			return nil
 		}
 		lastTargetPaths = me.TargetPaths
@@ -239,6 +244,10 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
 }
 
 func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath string) error {
+	if len(MountCacheDir) == 0 {
+		//if mount cache dir unset, disable remount
+		return nil
+	}
 	volumeMountCacheMtx.Lock()
 	defer volumeMountCacheMtx.Unlock()
 	delete(volumeMountCache.Volumes, volID)
@@ -250,6 +259,10 @@ func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath
 }
 
 func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
+	if len(MountCacheDir) == 0 {
+		//if mount cache dir unset, disable remount
+		return nil
+	}
 	volumeMountCacheMtx.Lock()
 	defer volumeMountCacheMtx.Unlock()
 
@@ -259,15 +272,14 @@ func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string
 		return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
 	}
 	volumeMountCache.Volumes[volID].TargetPaths[targetPath] = readOnly
-	me := volumeMountCache.Volumes[volID]
+	return mc.updateNodeCache(volID)
-	if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
-		klog.Errorf("mount-cache: node publish volume failed to store a cache entry for volume %s: %v", volID, err)
-		return err
-	}
-	return nil
 }
 
 func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
+	if len(MountCacheDir) == 0 {
+		//if mount cache dir unset, disable remount
+		return nil
+	}
 	volumeMountCacheMtx.Lock()
 	defer volumeMountCacheMtx.Unlock()
 
@@ -277,9 +289,16 @@ func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath stri
 		return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
 	}
 	delete(volumeMountCache.Volumes[volID].TargetPaths, targetPath)
+	return mc.updateNodeCache(volID)
+}
+
+func (mc *volumeMountCacheMap) updateNodeCache(volID string) error {
 	me := volumeMountCache.Volumes[volID]
-	if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
+	if err := volumeMountCache.NodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
-		klog.Errorf("mount-cache: node unpublish volume failed to store a cache entry for volume %s: %v", volID, err)
+		klog.Infof("mount-cache: metadata nofound, delete mount cache failed for volume %s", volID)
+	}
+	if err := mc.NodeCacheStore.Create(genVolumeMountCacheFileName(volID), me); err != nil {
+		klog.Errorf("mount-cache: mount cache failed to update for volume %s: %v", volID, err)
 		return err
 	}
 	return nil

pkg/util/nodecache.go

@@ -101,20 +101,6 @@ func decodeObj(filepath, pattern string, file os.FileInfo, destObj interface{})
 
 }
 
-func (nc *NodeCache) Update(identifier string, data interface{}) error {
-	file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
-	identifierTmp := identifier + ".creating"
-	fileTmp := path.Join(nc.BasePath, nc.CacheDir, identifierTmp+".json")
-	os.Remove(fileTmp)
-	if err := nc.Create(identifierTmp, data); err != nil {
-		return errors.Wrapf(err, "node-cache: failed to create metadata storage file %s\n", file)
-	}
-	if err := os.Rename(fileTmp, file); err != nil {
-		return errors.Wrapf(err, "node-cache: couldn't rename %s as %s", fileTmp, file)
-	}
-	return nil
-}
-
 // Create creates the metadata file in cache directory with identifier name
 func (nc *NodeCache) Create(identifier string, data interface{}) error {
 	file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")