Merge pull request #53 from ceph/devel

Sync rhs/ceph-csi:devel with ceph/ceph-csi:devel

charts/ceph-csi-rbd/values.yaml

@@ -284,10 +284,11 @@ storageClass:
   thickProvision: false
 
   # (required) RBD image features, CSI creates image with image-format 2
-  # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`
+  # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
-  # features. If `journaling` is enabled, must enable `exclusive-lock` too.
+  # `object-map`, `fast-diff` features. If `journaling` is enabled, must
-  # imageFeatures: layering,journaling,exclusive-lock
+  # enable `exclusive-lock` too.
-  imageFeatures: layering
+  # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff
+  imageFeatures: "layering"
 
   # (optional) Specifies whether to try other mounters in case if the current
   # mounter fails to mount the rbd image for any reason. True means fallback

docs/deploy-rbd.md

@@ -55,7 +55,7 @@ make image-cephcsi
 | `dataPool`                                                                                          | no                   | Ceph pool used for the data of the RBD images.                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
 | `volumeNamePrefix`                                                                                  | no                   | Prefix to use for naming RBD images (defaults to `csi-vol-`).                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 | `snapshotNamePrefix`                                                                                | no                   | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`).                                                                                                                                                                                                                                                                                                                                                                                                                                |
-| `imageFeatures`                                                                                     | yes                  | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. |
+| `imageFeatures`                                                                                     | yes                  | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. |
 | `tryOtherMounters`                                                                                  | no                   | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason                                                                                                                                                                                                                                                                                                                                                                                     |
 | `mapOptions`                                                                                        | no                   | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options.                                                                                                                                                                                                                                                                                               |
 | `unmapOptions`                                                                                      | no                   | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options.                                                                                                                                                                                                                                                                                           |

e2e/cephfs.go

@@ -315,33 +315,32 @@ var _ = Describe("cephfs", func() {
 			}
 			By("verify generic ephemeral volume support", func() {
 				// generic ephemeral volume support is beta since v1.21.
-				if !k8sVersionGreaterEquals(f.ClientSet, 1, 21) {
+				if k8sVersionGreaterEquals(f.ClientSet, 1, 21) {
-					Skip("generic ephemeral volume only supported from v1.21+")
+					err := createCephfsStorageClass(f.ClientSet, f, true, nil)
-				}
+					if err != nil {
-				err := createCephfsStorageClass(f.ClientSet, f, true, nil)
+						e2elog.Failf("failed to create CephFS storageclass: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to create CephFS storageclass: %v", err)
+					// create application
-				}
+					app, err := loadApp(appEphemeralPath)
-				// create application
+					if err != nil {
-				app, err := loadApp(appEphemeralPath)
+						e2elog.Failf("failed to load application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to load application: %v", err)
+					app.Namespace = f.UniqueName
-				}
+					err = createApp(f.ClientSet, app, deployTimeout)
-				app.Namespace = f.UniqueName
+					if err != nil {
-				err = createApp(f.ClientSet, app, deployTimeout)
+						e2elog.Failf("failed to create application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to create application: %v", err)
+					validateSubvolumeCount(f, 1, fileSystemName, subvolumegroup)
-				}
+					// delete pod
-				validateSubvolumeCount(f, 1, fileSystemName, subvolumegroup)
+					err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
-				// delete pod
+					if err != nil {
-				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+						e2elog.Failf("failed to delete application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to delete application: %v", err)
+					validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
-				}
+					err = deleteResource(cephFSExamplePath + "storageclass.yaml")
-				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+					if err != nil {
-				err = deleteResource(cephFSExamplePath + "storageclass.yaml")
+						e2elog.Failf("failed to delete CephFS storageclass: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to delete CephFS storageclass: %v", err)
 				}
 			})
 

e2e/rbd.go

@@ -373,31 +373,30 @@ var _ = Describe("RBD", func() {
 			}
 			By("verify generic ephemeral volume support", func() {
 				// generic ephemeral volume support is supported from 1.21
-				if !k8sVersionGreaterEquals(f.ClientSet, 1, 21) {
+				if k8sVersionGreaterEquals(f.ClientSet, 1, 21) {
-					Skip("generic ephemeral volume only supported from v1.21+")
+					// create application
-				}
+					app, err := loadApp(appEphemeralPath)
-				// create application
+					if err != nil {
-				app, err := loadApp(appEphemeralPath)
+						e2elog.Failf("failed to load application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to load application: %v", err)
+					app.Namespace = f.UniqueName
-				}
+					err = createApp(f.ClientSet, app, deployTimeout)
-				app.Namespace = f.UniqueName
+					if err != nil {
-				err = createApp(f.ClientSet, app, deployTimeout)
+						e2elog.Failf("failed to create application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to create application: %v", err)
+					// validate created backend rbd images
-				}
+					validateRBDImageCount(f, 1, defaultRBDPool)
-				// validate created backend rbd images
+					err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
-				validateRBDImageCount(f, 1, defaultRBDPool)
+					if err != nil {
-				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+						e2elog.Failf("failed to delete application: %v", err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to delete application: %v", err)
+					// validate created backend rbd images
-				}
+					validateRBDImageCount(f, 0, defaultRBDPool)
-				// validate created backend rbd images
+					// validate images in trash
-				validateRBDImageCount(f, 0, defaultRBDPool)
+					err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
-				// validate images in trash
+					if err != nil {
-				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
+						e2elog.Failf("failed to validate rbd images in pool %s trash: %v", defaultRBDPool, err)
-				if err != nil {
+					}
-					e2elog.Failf("failed to validate rbd images in pool %s trash: %v", defaultRBDPool, err)
 				}
 			})
 

examples/rbd/storageclass.yaml

@@ -34,10 +34,11 @@ parameters:
    # thickProvision: "false"
 
    # (required) RBD image features, CSI creates image with image-format 2
-   # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`
+   # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
-   # features. If `journaling` is enabled, must enable `exclusive-lock` too.
+   # `object-map`, `fast-diff` features. If `journaling` is enabled, must
-   # imageFeatures: layering,journaling,exclusive-lock
+   # enable `exclusive-lock` too.
-   imageFeatures: layering
+   # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff
+   imageFeatures: "layering"
 
    # (optional) Specifies whether to try other mounters in case if the current
    # mounter fails to mount the rbd image for any reason. True means fallback

internal/rbd/rbd_util.go

@@ -206,7 +206,15 @@ var (
 			needRbdNbd: false,
 		},
 		librbd.FeatureNameExclusiveLock: {
-			needRbdNbd: true,
+			needRbdNbd: false,
+		},
+		librbd.FeatureNameObjectMap: {
+			needRbdNbd: false,
+			dependsOn:  []string{librbd.FeatureNameExclusiveLock},
+		},
+		librbd.FeatureNameFastDiff: {
+			needRbdNbd: false,
+			dependsOn:  []string{librbd.FeatureNameObjectMap},
 		},
 		librbd.FeatureNameJournaling: {
 			needRbdNbd: true,

internal/rbd/rbd_util_test.go

@@ -88,21 +88,37 @@ func TestValidateImageFeatures(t *testing.T) {
 			false,
 			"",
 		},
+		{
+			"layering,exclusive-lock,object-map,fast-diff",
+			&rbdVolume{
+				Mounter: rbdDefaultMounter,
+			},
+			false,
+			"",
+		},
 		{
 			"layering,journaling",
 			&rbdVolume{
-				Mounter: rbdNbdMounter,
+				Mounter: rbdDefaultMounter,
 			},
 			true,
 			"feature journaling requires exclusive-lock to be set",
 		},
 		{
-			"layering,exclusive-lock,journaling",
+			"object-map,fast-diff",
 			&rbdVolume{
 				Mounter: rbdDefaultMounter,
 			},
 			true,
-			"feature exclusive-lock requires rbd-nbd for mounter",
+			"feature object-map requires exclusive-lock to be set",
+		},
+		{
+			"fast-diff",
+			&rbdVolume{
+				Mounter: rbdDefaultMounter,
+			},
+			true,
+			"feature fast-diff requires object-map to be set",
 		},
 		{
 			"layering,exclusive-lock,journaling",
@@ -110,7 +126,15 @@ func TestValidateImageFeatures(t *testing.T) {
 				Mounter: rbdDefaultMounter,
 			},
 			true,
-			"feature exclusive-lock requires rbd-nbd for mounter",
+			"feature journaling requires rbd-nbd for mounter",
+		},
+		{
+			"layering,exclusive-lock,journaling",
+			&rbdVolume{
+				Mounter: rbdDefaultMounter,
+			},
+			true,
+			"feature journaling requires rbd-nbd for mounter",
 		},
 		{
 			"layering,exclusive-loc,journaling",

internal/rbd/replicationcontrollerserver.go

@@ -300,23 +300,58 @@ func getOperationName(poolName string, optName operation) string {
 // createDummyImage creates a dummy image as a workaround for the rbd
 // scheduling problem.
 func createDummyImage(ctx context.Context, rbdVol *rbdVolume) error {
+	var err error
+	var imgName string
+
+	dummyImageOpsLock.Lock()
+	defer dummyImageOpsLock.Unlock()
 	optName := getOperationName(rbdVol.Pool, dummyImageCreated)
 	if _, ok := operationLock.Load(optName); !ok {
 		// create a dummy image
-		imgName, err := getDummyImageName(rbdVol.conn)
+		imgName, err = getDummyImageName(rbdVol.conn)
 		if err != nil {
 			return err
 		}
 		dummyVol := *rbdVol
 		dummyVol.RbdImageName = imgName
-		err = createImage(ctx, &dummyVol, dummyVol.conn.Creds)
+		f := []string{
-		if err != nil && !strings.Contains(err.Error(), "File exists") {
+			librbd.FeatureNameLayering,
-			return err
+			librbd.FeatureNameObjectMap,
+			librbd.FeatureNameExclusiveLock,
+			librbd.FeatureNameFastDiff,
+		}
+		features := librbd.FeatureSetFromNames(f)
+		dummyVol.imageFeatureSet = features
+		// create 1MiB dummy image. 1MiB=1048576 bytes
+		dummyVol.VolSize = 1048576
+		err = createImage(ctx, &dummyVol, dummyVol.conn.Creds)
+		if err != nil {
+			if strings.Contains(err.Error(), "File exists") {
+				err = repairDummyImage(ctx, &dummyVol)
+			}
+		}
+		if err == nil {
+			operationLock.Store(optName, true)
 		}
-		operationLock.Store(optName, true)
 	}
 
-	return nil
+	return err
+}
+
+// repairDummyImage deletes and recreates the dummy image.
+func repairDummyImage(ctx context.Context, dummyVol *rbdVolume) error {
+	// instead of checking the images features and than adding missing image
+	// features, updating the image size to 1Mib. We will delete the image
+	// and recreate it.
+
+	// deleting and recreating the dummy image will not impact anything as its
+	// a workaround to fix the scheduling problem.
+	err := deleteImage(ctx, dummyVol, dummyVol.conn.Creds)
+	if err != nil {
+		return err
+	}
+
+	return createImage(ctx, dummyVol, dummyVol.conn.Creds)
 }
 
 // tickleMirroringOnDummyImage disables and reenables mirroring on the dummy image, and sets a