From 034b1234781de9003ff9653b96d0206c54ac13f1 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 10 Feb 2020 14:21:05 +0530 Subject: [PATCH] Remove mount cache for cephfs PR #282 introduces the mount cache to solve cephfs fuse mount issue when cephfs plugin pod restarts .This is not working as intended. This PR removes the code for maintainability. Signed-off-by: Madhu Rajanna --- .../templates/nodeplugin-daemonset.yaml | 5 - cmd/cephcsi.go | 4 + .../cephfs/kubernetes/csi-cephfsplugin.yaml | 5 - docs/deploy-cephfs.md | 35 +- pkg/cephfs/driver.go | 8 - pkg/cephfs/mountcache.go | 330 ------------------ pkg/cephfs/mountcache_test.go | 38 -- pkg/cephfs/nodeserver.go | 16 - 8 files changed, 21 insertions(+), 420 deletions(-) delete mode 100644 pkg/cephfs/mountcache.go delete mode 100644 pkg/cephfs/mountcache_test.go diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 57b945a06..bc193ea43 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -85,7 +85,6 @@ spec: - "--v=5" - "--drivername=$(DRIVER_NAME)" - "--metadatastorage=k8s_configmap" - - "--mountcachedir=/mount-cache-dir" env: - name: POD_IP valueFrom: @@ -109,8 +108,6 @@ spec: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true volumeMounts: - - name: mount-cache-dir - mountPath: /mount-cache-dir - name: socket-dir mountPath: /csi - name: mountpoint-dir @@ -161,8 +158,6 @@ spec: {{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} {{- end }} volumes: - - name: mount-cache-dir - emptyDir: {} - name: socket-dir hostPath: path: {{ .Values.socketDir }} diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index ba42edc3e..11b30db23 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -60,6 +60,7 @@ func init() { flag.BoolVar(&conf.IsNodeServer, "nodeserver", false, "start cephcsi node server") // cephfs related flags + // marking this as deprecated, remove it in next major release flag.StringVar(&conf.MountCacheDir, "mountcachedir", "", "mount info cache save dir") flag.BoolVar(&conf.ForceKernelCephFS, "forcecephkernelclient", false, "enable Ceph Kernel clients on kernel < 4.17 which support quotas") @@ -172,6 +173,9 @@ func main() { driver.Run(&conf, cp) case cephfsType: + if conf.MountCacheDir != "" { + klog.Warning("mountcachedir option is deprecated") + } driver := cephfs.NewDriver() driver.Run(&conf, cp) diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml index bd210619f..5ff9b0f81 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml @@ -63,7 +63,6 @@ spec: - "--v=5" - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - - "--mountcachedir=/mount-cache-dir" - "--metricsport=8090" - "--metricspath=/metrics" - "--enablegrpcmetrics=false" @@ -84,8 +83,6 @@ spec: value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - - name: mount-cache-dir - mountPath: /mount-cache-dir - name: socket-dir mountPath: /csi - name: mountpoint-dir @@ -130,8 +127,6 @@ spec: mountPath: /csi imagePullPolicy: "IfNotPresent" volumes: - - name: mount-cache-dir - emptyDir: {} - name: socket-dir hostPath: path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/ diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index 89b3df598..cc67586c9 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -43,24 +43,23 @@ that should be resolved in v14.2.3. **Available command line arguments:** -| Option | Default value | Description | -| -------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | -| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | -| `--nodeid` | _empty_ | This node's ID | -| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | -| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. | -| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | -| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | -| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | -| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | -| `--metricsport` | `8080` | TCP port for /grpc metrics requests | -| `--metricspath` | `/metrics` | Path of prometheus endpoint where metrics will be available | -| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server | -| `--polltime` | `60s` | Time interval in between each poll | -| `--timeout` | `3s` | Probe timeout in seconds | -| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | -| `--forcecephkernelclient` | `false` | Force enabling Ceph Kernel clients for mounting on kernels < 4.17 | +| Option | Default value | Description | +| ------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | +| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | +| `--nodeid` | _empty_ | This node's ID | +| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | +| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | +| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | +| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | +| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | +| `--metricsport` | `8080` | TCP port for /grpc metrics requests | +| `--metricspath` | `/metrics` | Path of prometheus endpoint where metrics will be available | +| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server | +| `--polltime` | `60s` | Time interval in between each poll | +| `--timeout` | `3s` | Probe timeout in seconds | +| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | +| `--forcecephkernelclient` | `false` | Force enabling Ceph Kernel clients for mounting on kernels < 4.17 | **NOTE:** The parameter `-forcecephkernelclient` enables the Kernel CephFS mounter on kernels < 4.17. diff --git a/pkg/cephfs/driver.go b/pkg/cephfs/driver.go index 7d7c87927..b609e03de 100644 --- a/pkg/cephfs/driver.go +++ b/pkg/cephfs/driver.go @@ -115,14 +115,6 @@ func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { // Update namespace for storing keys into a specific namespace on RADOS, in the CephFS // metadata pool volJournal.SetNamespace(radosNamespace) - - initVolumeMountCache(conf.DriverName, conf.MountCacheDir) - if conf.MountCacheDir != "" { - if err := remountCachedVolumes(); err != nil { - klog.Warningf("failed to remount cached volumes: %v", err) - // ignore remount fail - } - } // Initialize default library driver fs.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID) diff --git a/pkg/cephfs/mountcache.go b/pkg/cephfs/mountcache.go deleted file mode 100644 index 8c85cea82..000000000 --- a/pkg/cephfs/mountcache.go +++ /dev/null @@ -1,330 +0,0 @@ -/* -Copyright 2019 The Ceph-CSI Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cephfs - -import ( - "context" - "encoding/base64" - "os" - "sync" - "syscall" - "time" - - "github.com/ceph/ceph-csi/pkg/util" - "github.com/pkg/errors" - "k8s.io/klog" -) - -type volumeMountCacheEntry struct { - DriverVersion string `json:"driverVersion"` - - VolumeID string `json:"volumeID"` - Mounter string `json:"mounter"` - Secrets map[string]string `json:"secrets"` - StagingPath string `json:"stagingPath"` - TargetPaths map[string]bool `json:"targetPaths"` - CreateTime time.Time `json:"createTime"` -} - -type volumeMountCacheMap struct { - volumes map[string]volumeMountCacheEntry - nodeCacheStore util.NodeCache -} - -var ( - volumeMountCachePrefix = "cephfs-mount-cache-" - volumeMountCache volumeMountCacheMap - volumeMountCacheMtx sync.Mutex -) - -func initVolumeMountCache(driverName, mountCacheDir string) { - volumeMountCache.volumes = make(map[string]volumeMountCacheEntry) - - volumeMountCache.nodeCacheStore.BasePath = mountCacheDir - volumeMountCache.nodeCacheStore.CacheDir = driverName - klog.Infof("mount-cache: name: %s, version: %s, mountCacheDir: %s", driverName, util.DriverVersion, mountCacheDir) -} - -func remountCachedVolumes() error { - if err := util.CreateMountPoint(volumeMountCache.nodeCacheStore.BasePath); err != nil { - klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.nodeCacheStore.BasePath, err) - return err - } - var remountFailCount, remountSuccCount int64 - me := &volumeMountCacheEntry{} - err := volumeMountCache.nodeCacheStore.ForAll(volumeMountCachePrefix, me, func(identifier string) error { - volID := me.VolumeID - if volOpts, vid, err := newVolumeOptionsFromVolID(context.TODO(), me.VolumeID, nil, decodeCredentials(me.Secrets)); err != nil { - if err, ok := err.(util.ErrKeyNotFound); ok { - klog.Infof("mount-cache: image key not found, assuming the volume %s to be already deleted (%v)", volID, err) - if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil { - klog.Infof("mount-cache: metadata not found, delete volume cache entry for volume %s", volID) - } - } - } else { - // update Mounter from mount cache - volOpts.Mounter = me.Mounter - if err := mountOneCacheEntry(volOpts, vid, me); err == nil { - remountSuccCount++ - volumeMountCache.volumes[me.VolumeID] = *me - klog.Infof("mount-cache: successfully remounted volume %s", volID) - } else { - remountFailCount++ - klog.Errorf("mount-cache: failed to remount volume %s", volID) - } - } - return nil - }) - if err != nil { - klog.Infof("mount-cache: metastore list cache fail %v", err) - return err - } - if remountFailCount > 0 { - klog.Infof("mount-cache: successfully remounted %d volumes, failed to remount %d volumes", remountSuccCount, remountFailCount) - } else { - klog.Infof("mount-cache: successfully remounted %d volumes", remountSuccCount) - } - return nil -} - -func mountOneCacheEntry(volOptions *volumeOptions, vid *volumeIdentifier, me *volumeMountCacheEntry) error { - volumeMountCacheMtx.Lock() - defer volumeMountCacheMtx.Unlock() - - var ( - err error - cr *util.Credentials - ) - volID := vid.VolumeID - - if volOptions.ProvisionVolume { - cr, err = util.NewAdminCredentials(decodeCredentials(me.Secrets)) - if err != nil { - return err - } - defer cr.DeleteCredentials() - - volOptions.RootPath, err = getVolumeRootPathCeph(context.TODO(), volOptions, cr, volumeID(vid.FsSubvolName)) - if err != nil { - return err - } - } else { - cr, err = util.NewUserCredentials(decodeCredentials(me.Secrets)) - if err != nil { - return err - } - defer cr.DeleteCredentials() - } - - err = cleanupMountPoint(me.StagingPath) - if err != nil { - klog.Infof("mount-cache: failed to cleanup volume mount point %s, remove it: %s %v", volID, me.StagingPath, err) - return err - } - - isMnt, err := util.IsMountPoint(me.StagingPath) - if err != nil { - isMnt = false - klog.Infof("mount-cache: failed to check volume mounted %s: %s %v", volID, me.StagingPath, err) - } - - if !isMnt { - m, err := newMounter(volOptions) - if err != nil { - klog.Errorf("mount-cache: failed to create mounter for volume %s: %v", volID, err) - return err - } - if err := m.mount(context.TODO(), me.StagingPath, cr, volOptions); err != nil { - klog.Errorf("mount-cache: failed to mount volume %s: %v", volID, err) - return err - } - } - - mountOptions := []string{"bind"} - for targetPath, readOnly := range me.TargetPaths { - if err := cleanupMountPoint(targetPath); err == nil { - if err := bindMount(context.TODO(), me.StagingPath, targetPath, readOnly, mountOptions); err != nil { - klog.Errorf("mount-cache: failed to bind-mount volume %s: %s %s %v %v", - volID, me.StagingPath, targetPath, readOnly, err) - } else { - klog.Infof("mount-cache: successfully bind-mounted volume %s: %s %s %v", - volID, me.StagingPath, targetPath, readOnly) - } - } - } - return nil -} - -func cleanupMountPoint(mountPoint string) error { - if _, err := os.Stat(mountPoint); err != nil { - if isCorruptedMnt(err) { - klog.Infof("mount-cache: corrupted mount point %s, need unmount", mountPoint) - err := execCommandErr(context.TODO(), "umount", mountPoint) - if err != nil { - klog.Infof("mount-cache: failed to umount %s %v", mountPoint, err) - // ignore error return err - } - } - } - if _, err := os.Stat(mountPoint); err != nil { - klog.Errorf("mount-cache: failed to stat mount point %s %v", mountPoint, err) - return err - } - return nil -} - -func isCorruptedMnt(err error) bool { - var underlyingError error - switch pe := err.(type) { - case nil: - return false - case *os.PathError: - underlyingError = pe.Err - case *os.LinkError: - underlyingError = pe.Err - case *os.SyscallError: - underlyingError = pe.Err - default: - return false - } - - CorruptedErrors := []error{ - syscall.ENOTCONN, syscall.ESTALE, syscall.EIO, syscall.EACCES} - - for _, v := range CorruptedErrors { - if underlyingError == v { - return true - } - } - return false -} - -func genVolumeMountCacheFileName(volID string) string { - cachePath := volumeMountCachePrefix + volID - return cachePath -} -func (mc *volumeMountCacheMap) isEnable() bool { - // if mount cache dir unset, disable state - return mc.nodeCacheStore.BasePath != "" -} - -func (mc *volumeMountCacheMap) nodeStageVolume(ctx context.Context, volID, stagingTargetPath, mounter string, secrets map[string]string) error { - if !mc.isEnable() { - return nil - } - volumeMountCacheMtx.Lock() - defer volumeMountCacheMtx.Unlock() - - lastTargetPaths := make(map[string]bool) - me, ok := volumeMountCache.volumes[volID] - if ok { - if me.StagingPath == stagingTargetPath { - klog.Warningf(util.Log(ctx, "mount-cache: node unexpected restage volume for volume %s"), volID) - return nil - } - lastTargetPaths = me.TargetPaths - klog.Warningf(util.Log(ctx, "mount-cache: node stage volume ignore last cache entry for volume %s"), volID) - } - - me = volumeMountCacheEntry{DriverVersion: util.DriverVersion} - - me.VolumeID = volID - me.Secrets = encodeCredentials(secrets) - me.StagingPath = stagingTargetPath - me.TargetPaths = lastTargetPaths - me.Mounter = mounter - - me.CreateTime = time.Now() - volumeMountCache.volumes[volID] = me - return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me) -} - -func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string) error { - if !mc.isEnable() { - return nil - } - volumeMountCacheMtx.Lock() - defer volumeMountCacheMtx.Unlock() - delete(volumeMountCache.volumes, volID) - return mc.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)) -} - -func (mc *volumeMountCacheMap) nodePublishVolume(ctx context.Context, volID, targetPath string, readOnly bool) error { - if !mc.isEnable() { - return nil - } - volumeMountCacheMtx.Lock() - defer volumeMountCacheMtx.Unlock() - - _, ok := volumeMountCache.volumes[volID] - if !ok { - return errors.New("mount-cache: node publish volume failed to find cache entry for volume") - } - volumeMountCache.volumes[volID].TargetPaths[targetPath] = readOnly - return mc.updateNodeCache(ctx, volID) -} - -func (mc *volumeMountCacheMap) nodeUnPublishVolume(ctx context.Context, volID, targetPath string) error { - if !mc.isEnable() { - return nil - } - volumeMountCacheMtx.Lock() - defer volumeMountCacheMtx.Unlock() - - _, ok := volumeMountCache.volumes[volID] - if !ok { - return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume") - } - delete(volumeMountCache.volumes[volID].TargetPaths, targetPath) - return mc.updateNodeCache(ctx, volID) -} - -func (mc *volumeMountCacheMap) updateNodeCache(ctx context.Context, volID string) error { - me := volumeMountCache.volumes[volID] - if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil { - klog.Infof(util.Log(ctx, "mount-cache: metadata not found, delete mount cache failed for volume %s"), volID) - } - return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me) -} - -func encodeCredentials(input map[string]string) (output map[string]string) { - output = make(map[string]string) - for key, value := range input { - nKey := base64.StdEncoding.EncodeToString([]byte(key)) - nValue := base64.StdEncoding.EncodeToString([]byte(value)) - output[nKey] = nValue - } - return output -} - -func decodeCredentials(input map[string]string) (output map[string]string) { - output = make(map[string]string) - for key, value := range input { - nKey, err := base64.StdEncoding.DecodeString(key) - if err != nil { - klog.Errorf("mount-cache: decode secret fail") - continue - } - nValue, err := base64.StdEncoding.DecodeString(value) - if err != nil { - klog.Errorf("mount-cache: decode secret fail") - continue - } - output[string(nKey)] = string(nValue) - } - return output -} diff --git a/pkg/cephfs/mountcache_test.go b/pkg/cephfs/mountcache_test.go deleted file mode 100644 index e27053cd4..000000000 --- a/pkg/cephfs/mountcache_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package cephfs - -import ( - "testing" -) - -func init() { -} - -func TestMountOneCacheEntry(t *testing.T) { -} - -func TestRemountHisMountedPath(t *testing.T) { -} - -func TestNodeStageVolume(t *testing.T) { -} - -func TestNodeUnStageVolume(t *testing.T) { -} - -func TestNodePublishVolume(t *testing.T) { -} - -func TestNodeUnpublishVolume(t *testing.T) { -} - -func TestEncodeDecodeCredentials(t *testing.T) { - secrets := make(map[string]string) - secrets["user_1"] = "value_1" - enSecrets := encodeCredentials(secrets) - deSecrets := decodeCredentials(enSecrets) - for key, value := range secrets { - if deSecrets[key] != value { - t.Errorf("key %s of credentials's value %s change after decode %s ", key, value, deSecrets[key]) - } - } -} diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index d3aa7cba5..63c8cfa7a 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -154,9 +154,6 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs klog.Errorf(util.Log(ctx, "failed to mount volume %s: %v"), volID, err) return status.Error(codes.Internal, err.Error()) } - if err := volumeMountCache.nodeStageVolume(ctx, req.GetVolumeId(), stagingTargetPath, volOptions.Mounter, req.GetSecrets()); err != nil { - klog.Warningf(util.Log(ctx, "mount-cache: failed to stage volume %s %s: %v"), volID, stagingTargetPath, err) - } return nil } @@ -209,10 +206,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Error(codes.Internal, err.Error()) } - if err = volumeMountCache.nodePublishVolume(ctx, volID, targetPath, req.GetReadonly()); err != nil { - klog.Warningf(util.Log(ctx, "mount-cache: failed to publish volume %s %s: %v"), volID, targetPath, err) - } - klog.Infof(util.Log(ctx, "cephfs: successfully bind-mounted volume %s to %s"), volID, targetPath) // #nosec - allow anyone to write inside the target path @@ -241,10 +234,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } defer ns.VolumeLocks.Release(volID) - if err = volumeMountCache.nodeUnPublishVolume(ctx, volID, targetPath); err != nil { - klog.Warningf(util.Log(ctx, "mount-cache: failed to unpublish volume %s %s: %v"), volID, targetPath, err) - } - // Unmount the bind-mount if err = unmountVolume(ctx, targetPath); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -275,11 +264,6 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag defer ns.VolumeLocks.Release(volID) stagingTargetPath := req.GetStagingTargetPath() - - if err = volumeMountCache.nodeUnStageVolume(volID); err != nil { - klog.Warningf(util.Log(ctx, "mount-cache: failed to unstage volume %s %s: %v"), volID, stagingTargetPath, err) - } - // Unmount the volume if err = unmountVolume(ctx, stagingTargetPath); err != nil { return nil, status.Error(codes.Internal, err.Error())