diff --git a/e2e/rbd.go b/e2e/rbd.go index 916426fe9..040c60b25 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -116,7 +116,6 @@ var _ = Describe("RBD", func() { By("create a PVC and Bind it to an app with normal user", func() { validateNormalUserPVCAccess(pvcPath, f) }) - // Skipping ext4 FS testing By("create a PVC and Bind it to an app with ext4 as the FS ", func() { deleteResource(rbdExamplePath + "storageclass.yaml") @@ -250,8 +249,16 @@ var _ = Describe("RBD", func() { e2elog.Logf("failed to resize PVC %v", err) Fail(err.Error()) } - } + deleteResource(rbdExamplePath + "storageclass.yaml") + createRBDStorageClass(f.ClientSet, f, map[string]string{"csi.storage.k8s.io/fstype": "xfs"}) + err = resizePVCAndValidateSize(pvcPath, appPath, f) + if err != nil { + e2elog.Logf("failed to resize PVC %v", err) + Fail(err.Error()) + + } + } }) By("Test unmount after nodeplugin restart", func() { diff --git a/e2e/utils.go b/e2e/utils.go index 90c1a72f1..fc1f811a8 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -834,12 +834,17 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s } pvcConditions := updatedPVC.Status.Conditions if len(pvcConditions) > 0 { - if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing { - return true, nil - } e2elog.Logf("pvc state %v", pvcConditions[0].Type) + if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing || pvcConditions[0].Type == v1.PersistentVolumeClaimFileSystemResizePending { + return false, nil + } } - return false, nil + + if updatedPVC.Status.Capacity[v1.ResourceStorage] != resource.MustParse(size) { + e2elog.Logf("current size in status %v,expected size %v", updatedPVC.Status.Capacity[v1.ResourceStorage], resource.MustParse(size)) + return false, nil + } + return true, nil }) } @@ -889,6 +894,10 @@ func resizePVCAndValidateSize(pvcPath, appPath string, f *framework.Framework) e return err } err = checkDirSize(app, f, &opt, expandSize, deployTimeout) + if err != nil { + return err + } + err = deletePVCAndApp("", f, resizePvc, app) return err } diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 237e6b387..5130f481f 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -535,7 +535,6 @@ func (cs *ControllerServer) doSnapshot(ctx context.Context, rbdSnap *rbdSnapshot err = status.Error(codes.Internal, err.Error()) } }() - err = protectSnapshot(ctx, rbdSnap, cr) if err != nil { klog.Errorf(util.Log(ctx, "failed to protect snapshot: %v"), err) @@ -645,7 +644,6 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS // ControllerExpandVolume expand RBD Volumes on demand based on resizer request func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil { klog.Warningf("invalid expand volume req: %v", protosanitizer.StripSecrets(req)) return nil, err @@ -653,14 +651,21 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi volID := req.GetVolumeId() if volID == "" { - return nil, status.Error(codes.InvalidArgument, "Volume ID cannot be empty") + return nil, status.Error(codes.InvalidArgument, "volume ID cannot be empty") } capRange := req.GetCapacityRange() if capRange == nil { - return nil, status.Error(codes.InvalidArgument, "CapacityRange cannot be empty") + return nil, status.Error(codes.InvalidArgument, "capacityRange cannot be empty") } + // lock out parallel requests against the same volume ID + if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired { + klog.Infof(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID) + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) + } + defer cs.VolumeLocks.Release(volID) + cr, err := util.NewUserCredentials(req.GetSecrets()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -670,28 +675,31 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi rbdVol := &rbdVolume{} err = genVolFromVolID(ctx, rbdVol, volID, cr) if err != nil { - return nil, status.Errorf(codes.NotFound, "Source Volume ID %s cannot found", volID) + if _, ok := err.(ErrImageNotFound); ok { + return nil, status.Errorf(codes.NotFound, "volume ID %s not found", volID) + } + return nil, status.Errorf(codes.Internal, err.Error()) } // always round up the request size in bytes to the nearest MiB/GiB volSize := util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes()) - klog.V(4).Infof(util.Log(ctx, "rbd volume %s/%s size is %vMiB,resizing to %vMiB"), rbdVol.Pool, rbdVol.RbdImageName, rbdVol.VolSize, util.RoundOffVolSize(volSize)) // resize volume if required + nodeExpansion := false if rbdVol.VolSize < volSize { - rbdVol.VolSize = util.RoundOffVolSize(volSize) - err = resizeRBDImage(ctx, rbdVol, cr) + volSizeVal := util.RoundOffVolSize(volSize) + klog.V(4).Infof(util.Log(ctx, "rbd volume %s/%s size is %v,resizing to %v"), rbdVol.Pool, rbdVol.RbdImageName, rbdVol.VolSize, volSize) + rbdVol.VolSize = volSize + nodeExpansion = true + err = resizeRBDImage(rbdVol, volSizeVal, cr) if err != nil { - klog.Errorf("failed to resize rbd image: %s/%s with error: %v", rbdVol.Pool, rbdVol.RbdImageName, err) + klog.Errorf(util.Log(ctx, "failed to resize rbd image: %s/%s with error: %v"), rbdVol.Pool, rbdVol.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) } - klog.V(4).Infof(util.Log(ctx, "successfully resized %s to %v"), volID, volSize) } - nodeExpansion := true return &csi.ControllerExpandVolumeResponse{ - CapacityBytes: volSize, + CapacityBytes: rbdVol.VolSize, NodeExpansionRequired: nodeExpansion, }, nil - } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index dc0e5966b..774d40fd7 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -538,45 +538,51 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag } func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { - volumeID := req.GetVolumeId() if volumeID == "" { - return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume ID must be provided") + return nil, status.Error(codes.InvalidArgument, "volume ID must be provided") } volumePath := req.GetVolumePath() if volumePath == "" { - return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume path must be provided") + return nil, status.Error(codes.InvalidArgument, "volume path must be provided") + } + + if acquired := ns.VolumeLocks.TryAcquire(volumeID); !acquired { + klog.Infof(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volumeID) + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) + } + defer ns.VolumeLocks.Release(volumeID) + + _, err := getVolumeName(volumeID) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) } diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()} - volumePath = volumePath + "/" + req.GetVolumeId() - devicePath, err := getDevicePath(ctx, diskMounter, volumePath) + devicePath, err := getDevicePath(ctx, volumePath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } // TODO check size and return success or error + volumePath += "/" + volumeID resizer := resizefs.NewResizeFs(diskMounter) ok, err := resizer.Resize(devicePath, volumePath) if !ok { return nil, fmt.Errorf("rbd: resize failed on path %s, error: %v", req.GetVolumePath(), err) } - return &csi.NodeExpandVolumeResponse{ - CapacityBytes: req.GetCapacityRange().GetRequiredBytes(), - }, nil + return &csi.NodeExpandVolumeResponse{}, nil } -func getDevicePath(ctx context.Context, mnt *mount.SafeFormatAndMount, stagingTargetPath string) (string, error) { - mointPoints, err := mnt.List() +func getDevicePath(ctx context.Context, volumePath string) (string, error) { + imgInfo, err := lookupRBDImageMetadataStash(volumePath) if err != nil { - return "", err + klog.Errorf(util.Log(ctx, "failed to find image metadata: %v"), err) } - for _, m := range mointPoints { - if strings.EqualFold(stagingTargetPath, m.Path) { - klog.V(3).Infof(util.Log(ctx, "found device %v for %v"), m.Device, stagingTargetPath) - return m.Device, nil - } + device, found := findDeviceMappingImage(ctx, imgInfo.Pool, imgInfo.ImageName, imgInfo.NbdAccess) + if found { + return device, nil } - return "", fmt.Errorf("failed to get device for stagingtarget path %v", stagingTargetPath) + return "", fmt.Errorf("failed to get device for stagingtarget path %v", volumePath) } // NodeGetCapabilities returns the supported capabilities of the node server diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index de2ea9e09..65bdfc565 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -815,14 +815,13 @@ func cleanupRBDImageMetadataStash(path string) error { } // resizeRBDImage resizes the given volume to new size -func resizeRBDImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error { +func resizeRBDImage(rbdVol *rbdVolume, newSize int64, cr *util.Credentials) error { var output []byte mon := rbdVol.Monitors image := rbdVol.RbdImageName - volSzMiB := fmt.Sprintf("%dM", rbdVol.VolSize) + volSzMiB := fmt.Sprintf("%dM", newSize) - klog.V(4).Infof(util.Log(ctx, "rbd: resize %s size %s using mon %s, pool %s "), image, volSzMiB, mon, rbdVol.Pool) args := []string{"resize", image, "--size", volSzMiB, "--pool", rbdVol.Pool, "--id", cr.ID, "-m", mon, "--keyfile=" + cr.KeyFile} output, err := execCommand("rbd", args)