ceph-csi/pkg/cephfs/mountcache.go

package cephfs

import (
	"encoding/base64"
	"os"
	"sync"
	"syscall"
	"time"

	"github.com/ceph/ceph-csi/pkg/util"
	"github.com/pkg/errors"
	"k8s.io/klog"
)

type volumeMountEntry struct {
	NodeID        string `json:"nodeID"`
	DriverName    string `json:"driverName"`
	DriverVersion string `json:"driverVersion"`

	Namespace string `json:"namespace"`

	VolumeID      string            `json:"volumeID"`
	Secrets       map[string]string `json:"secrets"`
	StagingPath   string            `json:"stagingPath"`
	TargetPaths   map[string]bool   `json:"targetPaths"`
	CreateTime    time.Time         `json:"createTime"`
	LastMountTime time.Time         `json:"lastMountTime"`
	LoadCount     uint64            `json:"loadCount"`
}

type volumeMountCacheMap struct {
	DriverName     string
	DriverVersion  string
	NodeID         string
	MountFailNum   int64
	MountSuccNum   int64
	Volumes        map[string]volumeMountEntry
	NodeCacheStore util.NodeCache
	MetadataStore  util.CachePersister
}

var (
	csiPersistentVolumeRoot = "/var/lib/kubelet/plugins/kubernetes.io/csi"
	volumeMountCachePrefix  = "cephfs-mount-cache-"
	volumeMountCache        volumeMountCacheMap
	volumeMountCacheMtx     sync.Mutex
)

func remountHisMountedPath(name string, v string, nodeID string, cachePersister util.CachePersister) error {
	volumeMountCache.Volumes = make(map[string]volumeMountEntry)
	volumeMountCache.NodeID = nodeID
	volumeMountCache.DriverName = name
	volumeMountCache.DriverVersion = v
	volumeMountCache.MountSuccNum = 0
	volumeMountCache.MountFailNum = 0

	volumeMountCache.MetadataStore = cachePersister

	volumeMountCache.NodeCacheStore.BasePath = PluginFolder
	volumeMountCache.NodeCacheStore.CacheDir = "volumes-mount-cache"

	if _, err := os.Stat(csiPersistentVolumeRoot); err != nil {
		klog.Infof("mount-cache: csi pv root path %s stat fail %v, may not in daemonset csi plugin, exit", csiPersistentVolumeRoot, err)
		return err
	}

	if err := os.MkdirAll(volumeMountCache.NodeCacheStore.BasePath, 0755); err != nil {
		klog.Fatalf("mount-cache: failed to create %s: %v", volumeMountCache.NodeCacheStore.BasePath, err)
		return err
	}
	me := &volumeMountEntry{}
	ce := &controllerCacheEntry{}
	err := volumeMountCache.NodeCacheStore.ForAll(volumeMountCachePrefix, me, func(identifier string) error {
		volID := me.VolumeID
		klog.Infof("mount-cache: load %v", me)
		if err := volumeMountCache.MetadataStore.Get(volID, ce); err != nil {
			if err, ok := err.(*util.CacheEntryNotFound); ok {
				klog.Infof("cephfs: metadata for volume %s not found, assuming the volume to be already deleted (%v)", volID, err)
				if err := volumeMountCache.NodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
					klog.Infof("mount-cache: metadata nofound, delete volume cache entry for volume %s", volID)
				}
			}
		} else {
			if err := mountOneCacheEntry(ce, me); err == nil {
				volumeMountCache.MountSuccNum++
				volumeMountCache.Volumes[me.VolumeID] = *me
			} else {
				volumeMountCache.MountFailNum++
			}
		}
		return nil
	})
	if err != nil {
		klog.Infof("mount-cache: metastore list cache fail %v", err)
		return err
	}
	if volumeMountCache.MountFailNum > volumeMountCache.MountSuccNum {
		return errors.New("mount-cache: too many volumes mount fail")
	}
	klog.Infof("mount-cache: succ remount %d volumes, fail remount %d volumes", volumeMountCache.MountSuccNum, volumeMountCache.MountFailNum)
	return nil
}

func mountOneCacheEntry(ce *controllerCacheEntry, me *volumeMountEntry) error {
	volumeMountCacheMtx.Lock()
	defer volumeMountCacheMtx.Unlock()

	var err error
	volID := ce.VolumeID
	volOptions := ce.VolOptions

	adminCr, err := getAdminCredentials(decodeCredentials(me.Secrets))
	if err != nil {
		return err
	}
	entity, err := getCephUser(&volOptions, adminCr, volID)
	if err != nil {
		klog.Infof("mount-cache: failed to get ceph user: %s %v", volID, me.StagingPath)
	}
	cr := entity.toCredentials()

	if volOptions.ProvisionVolume {
		volOptions.RootPath = getVolumeRootPathCeph(volID)
	}

	err = cleanupMountPoint(me.StagingPath)
	if err != nil {
		klog.Infof("mount-cache: failed to cleanup volume mount point %s, remove it: %s %v", volID, me.StagingPath, err)
		return err
	}

	isMnt, err := isMountPoint(me.StagingPath)
	if err != nil {
		isMnt = false
		klog.Infof("mount-cache: failed to check volume mounted %s: %s %v", volID, me.StagingPath, err)
	}

	if !isMnt {
		m, err := newMounter(&volOptions)
		if err != nil {
			klog.Errorf("mount-cache: failed to create mounter for volume %s: %v", volID, err)
			return err
		}
		if err := m.mount(me.StagingPath, cr, &volOptions); err != nil {
			klog.Errorf("mount-cache: failed to mount volume %s: %v", volID, err)
			return err
		}
	}
	for targetPath, readOnly := range me.TargetPaths {
		if err := cleanupMountPoint(targetPath); err == nil {
			if err := bindMount(me.StagingPath, targetPath, readOnly); err != nil {
				klog.Errorf("mount-cache: failed to bind-mount volume %s: %s %s %v %v",
					volID, me.StagingPath, targetPath, readOnly, err)
			} else {
				klog.Infof("mount-cache: succ bind-mount volume %s: %s %s %v",
					volID, me.StagingPath, targetPath, readOnly)
			}
		}
	}
	return nil
}

func cleanupMountPoint(mountPoint string) error {
	if _, err := os.Stat(mountPoint); err != nil {
		if IsCorruptedMnt(err) {
			klog.Infof("mount-cache: corrupted mount point %s, need unmount", mountPoint)
			err := execCommandErr("umount", mountPoint)
			if err != nil {
				klog.Infof("mount-cache: unmount %s fail %v", mountPoint, err)
				//ignore error return err
			}
		}
	}
	if _, err := os.Stat(mountPoint); err != nil {
		klog.Errorf("mount-cache: mount point %s stat fail %v", mountPoint, err)
		return err
	}
	return nil
}

func IsCorruptedMnt(err error) bool {
	if err == nil {
		return false
	}
	var underlyingError error
	switch pe := err.(type) {
	case nil:
		return false
	case *os.PathError:
		underlyingError = pe.Err
	case *os.LinkError:
		underlyingError = pe.Err
	case *os.SyscallError:
		underlyingError = pe.Err
	}

	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES
}

func genVolumeMountCacheFileName(volID string) string {
	cachePath := volumeMountCachePrefix + volID
	return cachePath
}

func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
	volumeMountCacheMtx.Lock()
	defer volumeMountCacheMtx.Unlock()

	lastTargetPaths := make(map[string]bool)
	me, ok := volumeMountCache.Volumes[volID]
	if ok {
		if me.StagingPath == stagingTargetPath {
			klog.Infof("mount-cache: node stage volume last cache entry for volume %s stagingTargetPath %s no equal %s",
				volID, me.StagingPath, stagingTargetPath)
			return nil
		}
		lastTargetPaths = me.TargetPaths
		klog.Warningf("mount-cache: node stage volume ignore last cache entry for volume %s", volID)
	}

	me = volumeMountEntry{NodeID: mc.NodeID, DriverName: mc.DriverName, DriverVersion: mc.DriverVersion}

	me.VolumeID = volID
	me.Secrets = encodeCredentials(secrets)
	me.StagingPath = stagingTargetPath
	me.TargetPaths = lastTargetPaths

	curTime := time.Now()
	me.CreateTime = curTime
	me.CreateTime = curTime
	me.LoadCount = 0
	volumeMountCache.Volumes[volID] = me
	if err := mc.NodeCacheStore.Create(genVolumeMountCacheFileName(volID), me); err != nil {
		klog.Errorf("mount-cache: node stage volume failed to store a cache entry for volume %s: %v", volID, err)
		return err
	}
	klog.Infof("mount-cache: node stage volume succ to store a cache entry for volume %s: %v", volID, me)
	return nil
}

func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string, stagingTargetPath string) error {
	volumeMountCacheMtx.Lock()
	defer volumeMountCacheMtx.Unlock()
	delete(volumeMountCache.Volumes, volID)
	if err := mc.NodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err != nil {
		klog.Infof("mount-cache: node unstage volume failed to delete cache entry for volume %s: %s %v", volID, stagingTargetPath, err)
		return err
	}
	return nil
}

func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
	volumeMountCacheMtx.Lock()
	defer volumeMountCacheMtx.Unlock()

	_, ok := volumeMountCache.Volumes[volID]
	if !ok {
		klog.Errorf("mount-cache: node publish volume failed to find cache entry for volume %s", volID)
		return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
	}
	volumeMountCache.Volumes[volID].TargetPaths[targetPath] = readOnly
	me := volumeMountCache.Volumes[volID]
	if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
		klog.Errorf("mount-cache: node publish volume failed to store a cache entry for volume %s: %v", volID, err)
		return err
	}
	return nil
}

func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
	volumeMountCacheMtx.Lock()
	defer volumeMountCacheMtx.Unlock()

	_, ok := volumeMountCache.Volumes[volID]
	if !ok {
		klog.Errorf("mount-cache: node unpublish volume failed to find cache entry for volume %s", volID)
		return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
	}
	delete(volumeMountCache.Volumes[volID].TargetPaths, targetPath)
	me := volumeMountCache.Volumes[volID]
	if err := mc.NodeCacheStore.Update(genVolumeMountCacheFileName(volID), me); err != nil {
		klog.Errorf("mount-cache: node unpublish volume failed to store a cache entry for volume %s: %v", volID, err)
		return err
	}
	return nil
}

func encodeCredentials(input map[string]string) (output map[string]string) {
	output = make(map[string]string)
	for key, value := range input {
		nKey := base64.StdEncoding.EncodeToString([]byte(key))
		nValue := base64.StdEncoding.EncodeToString([]byte(value))
		output[nKey] = nValue
	}
	return output
}

func decodeCredentials(input map[string]string) (output map[string]string) {
	output = make(map[string]string)
	for key, value := range input {
		nKey, err := base64.StdEncoding.DecodeString(key)
		if err != nil {
			klog.Errorf("mount-cache: decode secret fail")
			continue
		}
		nValue, err := base64.StdEncoding.DecodeString(value)
		if err != nil {
			klog.Errorf("mount-cache: decode secret fail")
			continue
		}
		output[string(nKey)] = string(nValue)
	}
	return output
}