rbd: perform resize of file system for static volume

For static volume, the user will manually mounts
already existing image as a volume to the application
pods. As its a rbd Image, if the PVC is of type
fileSystem the image will be mapped, formatted
and mounted on the node,
If the user resizes the image on the ceph cluster.
User cannot not automatically resize the filesystem
created on the rbd image. Even if deletes and
recreates the kubernetes objects, the new size
will not be visible on the node.

With this changes During the NodeStageVolumeRequest
the nodeplugin will check the size of the mapped rbd
image on the node using the devicePath. and also
the rbd image size on the ceph cluster.

If the size is not matching it will do the file
system resize on the node as part of the
NodeStageVolumeRequest RPC call.

The user need to do below operation to see new size
* Resize the rbd image in ceph cluster
* Scale down all the application pods using the static
PVC.
* Make sure no application pods which are using the
static PVC is running on a node.
* Scale up all the application pods.

Validate the new size in application pod mounted
volume.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

docs/static-pvc.md

@@ -6,6 +6,7 @@
     - [Create RBD static PV](#create-rbd-static-pv)
     - [RBD Volume Attributes in PV](#rbd-volume-attributes-in-pv)
     - [Create RBD static PVC](#create-rbd-static-pvc)
+    - [Resize RBD image](#resize-rbd-image)
   - [CephFS static PVC](#cephfs-static-pvc)
     - [Create CephFS subvolume](#create-cephfs-subvolume)
     - [Create CephFS static PV](#create-cephfs-static-pv)
@@ -124,6 +125,26 @@ $ kubectl create -f fs-static-pvc.yaml
 persistentvolumeclaim/fs-static-pvc created
 ```
 
+### Resize RBD image
+
+Let us resize the RBD image in ceph cluster
+
+```console
+rbd resize static-image --size=2048 --pool=replicapool
+```
+
+Once the rbd image is resized in the ceph cluster, update the PV size and PVC
+size to match the size of the rbd image.
+
+Now scale down the application pod which is using `cephfs-static-pvc` and scale
+up the application pod to resize the filesystem.
+
+**Note** If you have mounted same static PVC to multiple application pods, make
+sure you will scale down all the application pods and make sure no application
+pods using the static PVC is running on the node and scale up all the
+application pods again(this will trigger `NodeStageVolumeRequest` which will
+resize the filesystem for static volume).
+
 **Note** deleting PV and PVC does not removed the backend rbd image, user need to
 manually delete the rbd image if required
 

e2e/staticpvc.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	staticPVSize         = "4Gi"
+	staticPVNewSize      = "8Gi"
 	staticPVImageFeature = "layering"
 	monsPrefix           = "mons-"
 	imagePrefix          = "image-"
@@ -175,6 +176,11 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 		return err
 	}
 
+	app.Labels = make(map[string]string)
+	app.Labels[appKey] = appLabel
+	appOpt := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabel),
+	}
 	app.Namespace = namespace
 	app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
 	if checkImgFeat {
@@ -191,6 +197,14 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 		return err
 	}
 
+	// resize image only if the image is already mounted and formatted
+	if !checkImgFeat {
+		err = validateRBDStaticResize(f, app, &appOpt, pvc, rbdImageName)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to delete pvc: %w", err)
@@ -473,3 +487,36 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
 
 	return nil
 }
+
+func validateRBDStaticResize(
+	f *framework.Framework,
+	app *v1.Pod,
+	appOpt *metav1.ListOptions,
+	pvc *v1.PersistentVolumeClaim,
+	rbdImageName string) error {
+	// resize rbd image
+	size := staticPVNewSize
+	cmd := fmt.Sprintf(
+		"rbd resize %s --size=%s %s",
+		rbdImageName,
+		size,
+		rbdOptions(defaultRBDPool))
+
+	_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+	if err != nil {
+		return err
+	}
+	err = createApp(f.ClientSet, app, deployTimeout)
+	if err != nil {
+		return err
+	}
+	// check size for the filesystem type PVC
+	if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem {
+		err = checkDirSize(app, f, appOpt, size)
+		if err != nil {
+			return err
+		}
+	}
+
+	return deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+}

internal/rbd/nodeserver.go

@@ -453,6 +453,26 @@ func (ns *NodeServer) stageTransaction(
 	}
 	transaction.isMounted = true
 
+	// resize if its fileSystemType static volume.
+	if staticVol && !isBlock {
+		var ok bool
+		resizer := mount.NewResizeFs(utilexec.New())
+		ok, err = resizer.NeedResize(devicePath, stagingTargetPath)
+		if err != nil {
+			return transaction, status.Errorf(codes.Internal,
+				"Need resize check failed on devicePath %s and staingPath %s, error: %v",
+				devicePath,
+				stagingTargetPath,
+				err)
+		}
+		if ok {
+			ok, err = resizer.Resize(devicePath, stagingTargetPath)
+			if !ok {
+				return transaction, status.Errorf(codes.Internal,
+					"resize failed on path %s, error: %v", stagingTargetPath, err)
+			}
+		}
+	}
 	if !readOnly {
 		// #nosec - allow anyone to write inside the target path
 		err = os.Chmod(stagingTargetPath, 0o777)