mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-26 08:10:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

1. fix mountcache race conflict

Browse Source 
2. support user-defined cache dir
    3. if not define mountcachedir disable mountcache
This commit is contained in:
王怀宗 2019-03-27 16:04:58 +08:00
parent b318964af5
commit af330fe68e
3 changed files with 41 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/cephfs/main.go
View File

@ -31,6 +31,7 @@ var (
nodeID = flag.String("nodeid", "", "node id")
volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir")
)
func init() {
@ -49,6 +50,7 @@ func main() {
}
//update plugin name
cephfs.PluginFolder = cephfs.PluginFolder + *driverName
cephfs.MountCacheDir = *mountCacheDir
cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName)
if err != nil {

59
pkg/cephfs/mountcache.go
View File

@ -40,10 +40,10 @@ type volumeMountCacheMap struct {
}
var (
csiPersistentVolumeRoot = "/var/lib/kubelet/plugins/kubernetes.io/csi"
volumeMountCachePrefix = "cephfs-mount-cache-"
volumeMountCache volumeMountCacheMap
volumeMountCacheMtx sync.Mutex
MountCacheDir = ""
volumeMountCachePrefix = "cephfs-mount-cache-"
volumeMountCache volumeMountCacheMap
volumeMountCacheMtx sync.Mutex
)
func remountHisMountedPath(name string, v string, nodeID string, cachePersister util.CachePersister) error {
@ -56,16 +56,18 @@ func remountHisMountedPath(name string, v string, nodeID string, cachePersister
volumeMountCache.MetadataStore = cachePersister
volumeMountCache.NodeCacheStore.BasePath = PluginFolder
volumeMountCache.NodeCacheStore.CacheDir = "volumes-mount-cache"
volumeMountCache.NodeCacheStore.BasePath = MountCacheDir
volumeMountCache.NodeCacheStore.CacheDir = ""
if _, err := os.Stat(csiPersistentVolumeRoot); err != nil {
klog.Infof("mount-cache: csi pv root path %s stat fail %v, may not in daemonset csi plugin, exit", csiPersistentVolumeRoot, err)
return err
if len(MountCacheDir) == 0 {
//if mount cache dir unset, disable remount
klog.Infof("mount-cache: mountcachedir no define disalbe mount cache.")
return nil
}
klog.Infof("mount-cache: MountCacheDir: %s", MountCacheDir)
if err := os.MkdirAll(volumeMountCache.NodeCacheStore.BasePath, 0755); err != nil {
klog.Fatalf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
return err
}
me := &volumeMountEntry{}
@ -203,6 +205,10 @@ func genVolumeMountCacheFileName(volID string) string {
}
func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
if len(MountCacheDir) == 0 {
//if mount cache dir unset, disable remount
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
@ -210,8 +216,7 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
me, ok := volumeMountCache.Volumes[volID]
if ok {
if me.StagingPath == stagingTargetPath {
klog.Infof("mount-cache: node stage volume last cache entry for volume %s stagingTargetPath %s no equal %s",
volID, me.StagingPath, stagingTargetPath)
klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID)
return nil
}
lastTargetPaths = me.TargetPaths
@ -239,6 +244,10 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
}
func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath string) error {
if len(MountCacheDir) == 0 {
//if mount cache dir unset, disable remount
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
delete(volumeMountCache.Volumes, volID)
@ -250,6 +259,10 @@ func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath
}
func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
if len(MountCacheDir) == 0 {
//if mount cache dir unset, disable remount
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
@ -259,15 +272,14 @@ func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string
return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
}
volumeMountCache.Volumes[volID].TargetPaths[targetPath] = readOnly
me := volumeMountCache.Volumes[volID]
if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
klog.Errorf("mount-cache: node publish volume failed to store a cache entry for volume %s: %v", volID, err)
return err
}
return nil
return mc.updateNodeCache(volID)
}
func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
if len(MountCacheDir) == 0 {
//if mount cache dir unset, disable remount
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
@ -277,9 +289,16 @@ func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath stri
return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
}
delete(volumeMountCache.Volumes[volID].TargetPaths, targetPath)
return mc.updateNodeCache(volID)
}
func (mc *volumeMountCacheMap) updateNodeCache(volID string) error {
me := volumeMountCache.Volumes[volID]
if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
klog.Errorf("mount-cache: node unpublish volume failed to store a cache entry for volume %s: %v", volID, err)
if err := volumeMountCache.NodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
klog.Infof("mount-cache: metadata nofound, delete mount cache failed for volume %s", volID)
}
if err := mc.NodeCacheStore.Create(genVolumeMountCacheFileName(volID), me); err != nil {
klog.Errorf("mount-cache: mount cache failed to update for volume %s: %v", volID, err)
return err
}
return nil

14
pkg/util/nodecache.go
View File

@ -101,20 +101,6 @@ func decodeObj(filepath, pattern string, file os.FileInfo, destObj interface{})
}
func (nc *NodeCache) Update(identifier string, data interface{}) error {
file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
identifierTmp := identifier + ".creating"
fileTmp := path.Join(nc.BasePath, nc.CacheDir, identifierTmp+".json")
os.Remove(fileTmp)
if err := nc.Create(identifierTmp, data); err != nil {
return errors.Wrapf(err, "node-cache: failed to create metadata storage file %s\n", file)
}
if err := os.Rename(fileTmp, file); err != nil {
return errors.Wrapf(err, "node-cache: couldn't rename %s as %s", fileTmp, file)
}
return nil
}
// Create creates the metadata file in cache directory with identifier name
func (nc *NodeCache) Create(identifier string, data interface{}) error {
file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")