mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-22 14:20:19 +00:00
1. fix mountcache race conflict
2. support user-defined cache dir 3. if not define mountcachedir disable mountcache
This commit is contained in:
parent
b318964af5
commit
af330fe68e
@ -31,6 +31,7 @@ var (
|
||||
nodeID = flag.String("nodeid", "", "node id")
|
||||
volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
|
||||
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
|
||||
mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir")
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -49,6 +50,7 @@ func main() {
|
||||
}
|
||||
//update plugin name
|
||||
cephfs.PluginFolder = cephfs.PluginFolder + *driverName
|
||||
cephfs.MountCacheDir = *mountCacheDir
|
||||
|
||||
cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName)
|
||||
if err != nil {
|
||||
|
@ -40,10 +40,10 @@ type volumeMountCacheMap struct {
|
||||
}
|
||||
|
||||
var (
|
||||
csiPersistentVolumeRoot = "/var/lib/kubelet/plugins/kubernetes.io/csi"
|
||||
volumeMountCachePrefix = "cephfs-mount-cache-"
|
||||
volumeMountCache volumeMountCacheMap
|
||||
volumeMountCacheMtx sync.Mutex
|
||||
MountCacheDir = ""
|
||||
volumeMountCachePrefix = "cephfs-mount-cache-"
|
||||
volumeMountCache volumeMountCacheMap
|
||||
volumeMountCacheMtx sync.Mutex
|
||||
)
|
||||
|
||||
func remountHisMountedPath(name string, v string, nodeID string, cachePersister util.CachePersister) error {
|
||||
@ -56,16 +56,18 @@ func remountHisMountedPath(name string, v string, nodeID string, cachePersister
|
||||
|
||||
volumeMountCache.MetadataStore = cachePersister
|
||||
|
||||
volumeMountCache.NodeCacheStore.BasePath = PluginFolder
|
||||
volumeMountCache.NodeCacheStore.CacheDir = "volumes-mount-cache"
|
||||
volumeMountCache.NodeCacheStore.BasePath = MountCacheDir
|
||||
volumeMountCache.NodeCacheStore.CacheDir = ""
|
||||
|
||||
if _, err := os.Stat(csiPersistentVolumeRoot); err != nil {
|
||||
klog.Infof("mount-cache: csi pv root path %s stat fail %v, may not in daemonset csi plugin, exit", csiPersistentVolumeRoot, err)
|
||||
return err
|
||||
if len(MountCacheDir) == 0 {
|
||||
//if mount cache dir unset, disable remount
|
||||
klog.Infof("mount-cache: mountcachedir no define disalbe mount cache.")
|
||||
return nil
|
||||
}
|
||||
|
||||
klog.Infof("mount-cache: MountCacheDir: %s", MountCacheDir)
|
||||
if err := os.MkdirAll(volumeMountCache.NodeCacheStore.BasePath, 0755); err != nil {
|
||||
klog.Fatalf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
|
||||
klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
|
||||
return err
|
||||
}
|
||||
me := &volumeMountEntry{}
|
||||
@ -203,6 +205,10 @@ func genVolumeMountCacheFileName(volID string) string {
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
|
||||
if len(MountCacheDir) == 0 {
|
||||
//if mount cache dir unset, disable remount
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
@ -210,8 +216,7 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
|
||||
me, ok := volumeMountCache.Volumes[volID]
|
||||
if ok {
|
||||
if me.StagingPath == stagingTargetPath {
|
||||
klog.Infof("mount-cache: node stage volume last cache entry for volume %s stagingTargetPath %s no equal %s",
|
||||
volID, me.StagingPath, stagingTargetPath)
|
||||
klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID)
|
||||
return nil
|
||||
}
|
||||
lastTargetPaths = me.TargetPaths
|
||||
@ -239,6 +244,10 @@ func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath s
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath string) error {
|
||||
if len(MountCacheDir) == 0 {
|
||||
//if mount cache dir unset, disable remount
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
delete(volumeMountCache.Volumes, volID)
|
||||
@ -250,6 +259,10 @@ func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
|
||||
if len(MountCacheDir) == 0 {
|
||||
//if mount cache dir unset, disable remount
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
@ -259,15 +272,14 @@ func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string
|
||||
return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
|
||||
}
|
||||
volumeMountCache.Volumes[volID].TargetPaths[targetPath] = readOnly
|
||||
me := volumeMountCache.Volumes[volID]
|
||||
if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
|
||||
klog.Errorf("mount-cache: node publish volume failed to store a cache entry for volume %s: %v", volID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return mc.updateNodeCache(volID)
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
|
||||
if len(MountCacheDir) == 0 {
|
||||
//if mount cache dir unset, disable remount
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
@ -277,9 +289,16 @@ func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath stri
|
||||
return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
|
||||
}
|
||||
delete(volumeMountCache.Volumes[volID].TargetPaths, targetPath)
|
||||
return mc.updateNodeCache(volID)
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) updateNodeCache(volID string) error {
|
||||
me := volumeMountCache.Volumes[volID]
|
||||
if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
|
||||
klog.Errorf("mount-cache: node unpublish volume failed to store a cache entry for volume %s: %v", volID, err)
|
||||
if err := volumeMountCache.NodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
|
||||
klog.Infof("mount-cache: metadata nofound, delete mount cache failed for volume %s", volID)
|
||||
}
|
||||
if err := mc.NodeCacheStore.Create(genVolumeMountCacheFileName(volID), me); err != nil {
|
||||
klog.Errorf("mount-cache: mount cache failed to update for volume %s: %v", volID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
@ -101,20 +101,6 @@ func decodeObj(filepath, pattern string, file os.FileInfo, destObj interface{})
|
||||
|
||||
}
|
||||
|
||||
func (nc *NodeCache) Update(identifier string, data interface{}) error {
|
||||
file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
|
||||
identifierTmp := identifier + ".creating"
|
||||
fileTmp := path.Join(nc.BasePath, nc.CacheDir, identifierTmp+".json")
|
||||
os.Remove(fileTmp)
|
||||
if err := nc.Create(identifierTmp, data); err != nil {
|
||||
return errors.Wrapf(err, "node-cache: failed to create metadata storage file %s\n", file)
|
||||
}
|
||||
if err := os.Rename(fileTmp, file); err != nil {
|
||||
return errors.Wrapf(err, "node-cache: couldn't rename %s as %s", fileTmp, file)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create creates the metadata file in cache directory with identifier name
|
||||
func (nc *NodeCache) Create(identifier string, data interface{}) error {
|
||||
file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json")
|
||||
|
Loading…
Reference in New Issue
Block a user